diff --git a/.appveyor.yml b/.appveyor.yml index 559431e2f9421f4cf1bfe64f1e2e9ee66b7bead5..fe4816688b43276a4a9ca7b911b39a43b8fc2141 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,6 +3,7 @@ os: Visual Studio 2015 environment: matrix: - ARCH: amd64 + - ARCH: x86 clone_folder: c:\dev\TDengine clone_depth: 1 @@ -23,6 +24,7 @@ notifications: - provider: Email to: - sangshuduo@gmail.com + on_build_success: true on_build_failure: true on_build_status_changed: true diff --git a/.gitignore b/.gitignore index e6e327327c2a72bc2262d9b2923314c950c78cf6..1ff11080569e9312369f6e9c00463e25853fd38b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ tests/hdfs/ nmake/ sln/ hdfs/ -c/ taoshebei/ taosdalipu/ Target/ @@ -79,3 +78,15 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ + +# Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* +TAGS diff --git a/.gitmodules b/.gitmodules index 156226d54486c17e64b9c514e47e3a7dc3fe6942..74afbbf9973abec6423633b848181b349de4ed6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,9 @@ [submodule "src/connector/grafanaplugin"] path = src/connector/grafanaplugin url = https://github.com/taosdata/grafanaplugin +[submodule "src/connector/hivemq-tdengine-extension"] + path = src/connector/hivemq-tdengine-extension + url = https://github.com/huskar-t/hivemq-tdengine-extension.git [submodule "tests/examples/rust"] path = tests/examples/rust url = https://github.com/songtianyi/tdengine-rust-bindings.git -[submodule "src/connector/hivemq-tdengine-extension"] - path = src/connector/hivemq-tdengine-extension - url = https://github.com/huskar-t/hivemq-tdengine-extension.git \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index eb69370418a9c83c7b8bfe5daa1d6ead19150243..d814a465e67468fc05c2d03b62c092a9c5130e22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ matrix: - psmisc - unixodbc - unixodbc-dev + - mono-complete before_script: - export TZ=Asia/Harbin @@ -59,6 +60,18 @@ matrix: pip3 install guppy3 pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ + cd ${TRAVIS_BUILD_DIR}/tests/examples/C#/taosdemo + mcs -out:taosdemo *.cs || travis_terminate $? + pkill -TERM -x taosd + fuser -k -n tcp 6030 + sleep 1 + ${TRAVIS_BUILD_DIR}/debug/build/bin/taosd -c ${TRAVIS_BUILD_DIR}/debug/test/cfg > /dev/null & + sleep 5 + mono taosdemo -Q DEFAULT -y || travis_terminate $? + pkill -KILL -x taosd + fuser -k -n tcp 6030 + sleep 1 + cd ${TRAVIS_BUILD_DIR}/tests ./test-all.sh smoke || travis_terminate $? sleep 1 @@ -74,6 +87,7 @@ matrix: ./valgrind-test.sh 2>&1 > mem-error-out.log sleep 1 + # Color setting RED='\033[0;31m' GREEN='\033[1;32m' @@ -156,8 +170,9 @@ matrix: packages: - build-essential - cmake + - binutils-2.26 env: - - DESC="trusty/gcc-4.8 build" + - DESC="trusty/gcc-4.8/bintuils-2.26 build" before_script: - export TZ=Asia/Harbin @@ -168,7 +183,7 @@ matrix: script: - cmake .. > /dev/null - - make + - export PATH=/usr/lib/binutils-2.26/bin:$PATH && make - os: linux dist: bionic @@ -200,7 +215,7 @@ matrix: dist: bionic language: c compiler: clang - env: DESC="linux/clang build" + env: DESC="arm64 linux/clang build" git: - depth: 1 @@ -227,7 +242,7 @@ matrix: - os: linux arch: arm64 - dist: trusty + dist: xenial language: c git: - depth: 1 @@ -238,7 +253,7 @@ matrix: - build-essential - cmake env: - - DESC="trusty/gcc-4.8 build" + - DESC="arm64 xenial build" before_script: - export TZ=Asia/Harbin @@ -255,19 +270,20 @@ matrix: fi - make > /dev/null -# - os: osx -# language: c -# compiler: clang -# env: DESC="mac/clang build" -# git: -# - depth: 1 -# addons: -# homebrew: -# - cmake -# -# script: -# - cd ${TRAVIS_BUILD_DIR} -# - mkdir debug -# - cd debug -# - cmake .. > /dev/null -# - make > /dev/null + - os: osx + osx_image: xcode11.4 + language: c + compiler: clang + env: DESC="mac/clang build" + git: + - depth: 1 + addons: + homebrew: + - cmake + + script: + - cd ${TRAVIS_BUILD_DIR} + - mkdir debug + - cd debug + - cmake .. > /dev/null + - make > /dev/null diff --git a/CMakeLists.txt b/CMakeLists.txt index 588526c28614f753f6ff70e898f13f3fd65318db..7bb36fe1b001473cf5641ad195959581affeb2cb 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,9 +13,9 @@ ENDIF () SET(TD_ACCOUNT FALSE) SET(TD_ADMIN FALSE) SET(TD_GRANT FALSE) -SET(TD_SYNC TRUE) -SET(TD_MQTT TRUE) +SET(TD_MQTT FALSE) SET(TD_TSDB_PLUGINS FALSE) +SET(TD_STORAGE FALSE) SET(TD_COVER FALSE) SET(TD_MEM_CHECK FALSE) @@ -30,6 +30,11 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) + +IF (TD_WINDOWS OR TD_DARWIN) + SET(TD_SOMODE_STATIC TRUE) +ENDIF () + INCLUDE(cmake/define.inc) INCLUDE(cmake/env.inc) INCLUDE(cmake/version.inc) diff --git a/Jenkinsfile b/Jenkinsfile index edbe11d428d0381a81f64bbc823b5b51a5edf199..8d2429c137414fdb63260f94c0c99a6d264d8c48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,230 +1,285 @@ +import hudson.model.Result +import jenkins.model.CauseOfInterruption +properties([pipelineTriggers([githubPush()])]) +node { + git url: 'https://github.com/taosdata/TDengine.git' +} + +def skipstage=0 +def abortPreviousBuilds() { + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() + + for (build in builds) { + if (!build.isBuilding()) { + continue; + } + + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } + + build.doKill() //doTerm(),doKill(),doTerm() + } +} +//abort previous build +abortPreviousBuilds() +def abort_previous(){ + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) +} +def pre_test(){ + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + sudo rmtaos + ''' + } + sh ''' + + cd ${WKC} + git checkout develop + git reset --hard HEAD~10 >/dev/null + git pull + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0 + cd ${WK} + git reset --hard HEAD~10 + git checkout develop + git pull + cd ${WK} + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake .. > /dev/null + make > /dev/null + make install > /dev/null + cd ${WKC}/tests + ''' + return 1 +} + pipeline { agent none + environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDinternal/community' } - + stages { + stage('pre_build'){ + agent{label 'master'} + when { + changeRequest() + } + steps { + sh''' + cp -r ${WORKSPACE} ${WORKSPACE}.tes + cd ${WORKSPACE}.tes + git checkout develop + git pull + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + ''' + script{ + env.skipstage=sh(script:"cd ${WORKSPACE}.tes && git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) + } + println env.skipstage + sh''' + rm -rf ${WORKSPACE}.tes + ''' + } + } + stage('Parallel test stage') { + //only build pr + when { + changeRequest() + expression { + env.skipstage != 0 + } + } parallel { - stage('pytest') { - agent{label '184'} + stage('python_1_s1') { + agent{label 'p1'} steps { - sh ''' - date - cd ${WKC} - git reset --hard - git checkout develop - git pull - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - date - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null - make install > /dev/null - cd ${WKC}/tests - #./test-all.sh smoke - ./test-all.sh pytest - date''' + + pre_test() + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh p1 + date''' + } + } } - stage('test_b1') { - agent{label 'master'} + stage('python_2_s5') { + agent{label 'p2'} steps { - sh ''' - cd ${WKC} - git reset --hard - git checkout develop - git pull - - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - date - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null - cd ${WKC}/tests - #./test-all.sh smoke - ./test-all.sh b1 - date''' + + pre_test() + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh p2 + date''' + } + } + } + stage('python_3_s6') { + agent{label 'p3'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh p3 + date''' + } + } + } + stage('test_b1_s2') { + agent{label 'b1'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + cd ${WKC}/tests + ./test-all.sh b1fq + date''' + } } } - stage('test_crash_gen') { - agent{label "185"} + stage('test_crash_gen_s3') { + agent{label "b2"} + steps { - sh ''' - cd ${WKC} - git reset --hard - git checkout develop - git pull - - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null - cd ${WKC}/tests/pytest - ''' + pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest ./crash_gen.sh -a -p -t 4 -s 2000 ''' } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/tests/pytest - ./handle_crash_gen_val_log.sh - ''' - } + sh ''' - date - cd ${WKC}/tests - ./test-all.sh b2 - date + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_crash_gen_val_log.sh ''' + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b2fq + date + ''' + } + } } - stage('test_valgrind') { - agent{label "186"} + stage('test_valgrind_s4') { + agent{label "b3"} steps { - sh ''' - cd ${WKC} - git reset --hard - git checkout develop - git pull - - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - date - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null - cd ${WKC}/tests/pytest - ./valgrind-test.sh 2>&1 > mem-error-out.log - ./handle_val_log.sh - - date - cd ${WKC}/tests - ./test-all.sh b3 - date''' - } - } - stage('connector'){ - agent{label "release"} - steps{ - sh''' - cd ${WORKSPACE} - git checkout develop - ''' + pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' - cd ${WORKSPACE}/tests/gotest - bash batchtest.sh - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker - python3 PythonChecker.py + cd ${WKC}/tests/pytest + ./valgrind-test.sh 2>&1 > mem-error-out.log + ./handle_val_log.sh ''' + } + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b3fq + date''' } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single >/dev/null - java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - ''' + } + } + stage('test_b4_s7') { + agent{label 'b4'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b4fq + cd ${WKC}/tests + ./test-all.sh p4 + cd ${WKC}/tests + ./test-all.sh full jdbc + cd ${WKC}/tests + ./test-all.sh full unit + date''' } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# - dotnet run - ''' + } + } + stage('test_b5_s8') { + agent{label 'b5'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b5fq + date''' } - - } - } - stage('arm64_build'){ - agent{label 'arm64'} - steps{ - sh ''' - cd ${WK} - git fetch - git checkout develop - git pull - cd ${WKC} - git fetch - git checkout develop - git pull - git submodule update - cd ${WKC}/packaging - ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0 - - ''' - } + } + } + stage('test_b6_s9') { + agent{label 'b6'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b6fq + date''' } - stage('arm32_build'){ - agent{label 'arm32'} - steps{ - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WK} - git fetch - git checkout develop - git pull - cd ${WKC} - git fetch - git checkout develop - git pull - git submodule update - cd ${WKC}/packaging - ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0 - - ''' - } - - } + } + } + stage('test_b7_s10') { + agent{label 'b7'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b7fq + date''' } - } + } + } } - } - post { + } + post { success { emailext ( - subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", + body: """ @@ -240,29 +295,29 @@ pipeline { - ''', - to: "yqliu@taosdata.com,pxiao@taosdata.com", + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", from: "support@taosdata.com" ) } failure { emailext ( - subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL", + body: """ @@ -278,24 +333,25 @@ pipeline { - ''', - to: "yqliu@taosdata.com,pxiao@taosdata.com", + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", from: "support@taosdata.com" ) } } -} \ No newline at end of file + +} diff --git a/README.md b/README.md index 36436dd549e22f58caefd82c4a9be4e8aed8d869..489b6d0a4e12db06c9f2c1274e7a87f22cf76655 100644 --- a/README.md +++ b/README.md @@ -33,11 +33,17 @@ To build TDengine, use [CMake](https://cmake.org/) 3.5 or higher versions in the ## Install tools -### Ubuntu & Debian: +### Ubuntu 16.04 and above & Debian: ```bash sudo apt-get install -y gcc cmake build-essential git ``` +### Ubuntu 14.04: +```bash +sudo apt-get install -y gcc cmake3 build-essential git binutils-2.26 +export PATH=/usr/lib/binutils-2.26/bin:$PATH +``` + To compile and package the JDBC driver source code, you should have a Java jdk-8 or higher and Apache Maven 2.7 or higher installed. To install openjdk-8: ```bash @@ -51,9 +57,7 @@ sudo apt-get install -y maven ### Centos 7: ```bash -sudo yum install -y gcc gcc-c++ make cmake3 epel-release git -sudo yum remove -y cmake -sudo ln -s /usr/bin/cmake3 /usr/bin/cmake +sudo yum install -y gcc gcc-c++ make cmake git ``` To install openjdk-8: @@ -104,7 +108,8 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -To compile on an ARM processor (aarch64 or aarch32), please add option CPUTYPE as below: +TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 and arm32 platform. +You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: aarch64: ```bash @@ -120,44 +125,57 @@ cmake .. -DCPUTYPE=aarch32 && cmake --build . If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "x86_amd64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. -``` +```cmd mkdir debug && cd debug "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < x86_amd64 | x86 > cmake .. -G "NMake Makefiles" nmake ``` -If you use the Visual Studio 2019, please open a command window by executing "cmd.exe". +If you use the Visual Studio 2019 or 2017: + +please open a command window by executing "cmd.exe". Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. -``` + +```cmd mkdir debug && cd debug "c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > cmake .. -G "NMake Makefiles" nmake ``` -Or, you can open a command window by clicking Visual Studio 2019 menu "Tools -> Command Line -> Developer Command Prompt" or "Tools -> Command Line -> Developer PowerShell" then execute commands as follows: -``` +Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: +```cmd mkdir debug && cd debug cmake .. -G "NMake Makefiles" nmake ``` +### On Mac OS X platform + +Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. + +```shell +mkdir debug && cd debug +cmake .. && cmake --build . +``` + # Quick Run + # Quick Run To quickly start a TDengine server after building, run the command below in terminal: -```cmd +```bash ./build/bin/taosd -c test/cfg ``` In another terminal, use the TDengine shell to connect the server: -``` +```bash ./build/bin/taos -c test/cfg ``` option "-c test/cfg" specifies the system configuration file directory. # Installing After building successfully, TDengine can be installed by: -```cmd +```bash make install ``` Users can find more information about directories installed on the system in the [directory and files](https://www.taosdata.com/en/documentation/administrator/#Directory-and-Files) section. It should be noted that installing from source code does not configure service management for TDengine. diff --git a/alert/go.mod b/alert/go.mod index 01c557d5646714869ad263a9c8fe2fa262fe9a43..43920f1f1aea2a95d02fbaddeab8ebc80ad1e81f 100644 --- a/alert/go.mod +++ b/alert/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/jmoiron/sqlx v1.2.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible - github.com/taosdata/driver-go v0.0.0-20200727182616-1a3b1941c206 + github.com/taosdata/driver-go v0.0.0-20201113094317-050667e5b4d0 go.uber.org/zap v1.14.1 google.golang.org/appengine v1.6.5 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c diff --git a/cmake/define.inc b/cmake/define.inc index 6e64c2709abee366856a61916c6905363ea2002d..ae90410f2d6873b60ee0e355f18462983e615545 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -13,10 +13,6 @@ IF (TD_GRANT) ADD_DEFINITIONS(-D_GRANT) ENDIF () -IF (TD_SYNC) - ADD_DEFINITIONS(-D_SYNC) -ENDIF () - IF (TD_MQTT) ADD_DEFINITIONS(-D_MQTT) ENDIF () @@ -25,6 +21,10 @@ IF (TD_TSDB_PLUGINS) ADD_DEFINITIONS(-D_TSDB_PLUGINS) ENDIF () +IF (TD_STORAGE) + ADD_DEFINITIONS(-D_STORAGE) +ENDIF () + IF (TD_GODLL) ADD_DEFINITIONS(-D_TD_GO_DLL_) ENDIF () @@ -49,7 +49,7 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) MESSAGE(STATUS "linux64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () @@ -57,7 +57,7 @@ IF (TD_LINUX_32) ADD_DEFINITIONS(-D_TD_LINUX_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "linux32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_64) @@ -66,7 +66,7 @@ IF (TD_ARM_64) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_32) @@ -74,21 +74,21 @@ IF (TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") ENDIF () IF (TD_MIPS_64) ADD_DEFINITIONS(-D_TD_MIPS_64_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_MIPS_32) ADD_DEFINITIONS(-D_TD_MIPS_32_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_APLHINE) @@ -109,8 +109,8 @@ IF (TD_LINUX) MESSAGE(STATUS "set ningsi macro to true") ENDIF () - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0 -Wno-unused-variable -Wunused-but-set-variable") + SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") + SET(RELEASE_FLAGS "-O3 -Wno-error") IF (${COVER} MATCHES "true") MESSAGE(STATUS "Test coverage mode, add extra flags") @@ -129,9 +129,11 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "darwin64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") + SET(RELEASE_FLAGS "-Og") + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) ENDIF () IF (TD_WINDOWS) @@ -143,8 +145,11 @@ IF (TD_WINDOWS) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) + SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") + ENDIF () SET(DEBUG_FLAGS "/Zi /W3 /GL") - SET(RELEASE_FLAGS "/W0 /GL") + SET(RELEASE_FLAGS "/W0 /O3 /GL") ENDIF () INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) diff --git a/cmake/env.inc b/cmake/env.inc index 18a6fea51d7ab4c6a4068efafc478360737df9be..efcc9961767aec8555897e7a96fec91356017659 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -41,8 +41,10 @@ SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FL # SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") IF (${CMAKE_BUILD_TYPE} MATCHES "Debug") + SET(CMAKE_BUILD_TYPE "Debug") MESSAGE(STATUS "Build Debug Version") ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release") + SET(CMAKE_BUILD_TYPE "Release") MESSAGE(STATUS "Build Release Version") ELSE () IF (TD_WINDOWS) diff --git a/cmake/input.inc b/cmake/input.inc index 1ef2045f57e5c9df94b6fe8284930711e8624a8a..e8324887a051ae6ac27eff6b748c824d1f0a3fa5 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -47,11 +47,6 @@ IF (${MQTT} MATCHES "false") MESSAGE(STATUS "build without mqtt module") ENDIF () -IF (${SYNC} MATCHES "false") - SET(TD_SYNC FALSE) - MESSAGE(STATUS "build without sync module") -ENDIF () - IF (${RANDOM_FILE_FAIL} MATCHES "true") SET(TD_RANDOM_FILE_FAIL TRUE) MESSAGE(STATUS "build with random-file-fail enabled") diff --git a/cmake/install.inc b/cmake/install.inc index 8418612d4c54d7f7643e190da0af81064197b5d1..0ea79589caef1bf3ec72f4234f99e87328759c33 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.21-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/cmake/platform.inc b/cmake/platform.inc index 889f6c73cf2be982df7f1db207114e58d6af3e92..dcd0183e27b58f2ebd22c584fab359e025044cba 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -78,29 +78,58 @@ ELSE() EXIT () ENDIF () -# if generate ARM version: -# cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 -IF (${CPUTYPE} MATCHES "aarch32") - SET(TD_LINUX TRUE) - SET(TD_LINUX_32 FALSE) - SET(TD_ARM_32 TRUE) - MESSAGE(STATUS "input cpuType: aarch32") -ELSEIF (${CPUTYPE} MATCHES "aarch64") - SET(TD_LINUX TRUE) - SET(TD_LINUX_64 FALSE) - SET(TD_ARM_64 TRUE) - MESSAGE(STATUS "input cpuType: aarch64") -ELSEIF (${CPUTYPE} MATCHES "mips64") - SET(TD_LINUX TRUE) - SET(TD_LINUX_64 FALSE) - SET(TD_MIPS_64 TRUE) - MESSAGE(STATUS "input cpuType: mips64") -ELSEIF (${CPUTYPE} MATCHES "x64") - MESSAGE(STATUS "input cpuType: x64") -ELSEIF (${CPUTYPE} MATCHES "x86") - MESSAGE(STATUS "input cpuType: x86") +IF ("${CPUTYPE}" STREQUAL "") + MESSAGE(STATUS "The current platform " ${CMAKE_SYSTEM_PROCESSOR} " is detected") + + IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") + MESSAGE(STATUS "The current platform is amd64") + MESSAGE(STATUS "Set CPUTYPE to x64") + SET(CPUTYPE "x64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") + MESSAGE(STATUS "The current platform is x86") + MESSAGE(STATUS "Set CPUTYPE to x86") + SET(CPUTYPE "x32") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") + MESSAGE(STATUS "Set CPUTYPE to aarch32") + SET(CPUTYPE "aarch32") + MESSAGE(STATUS "Set CPUTYPE to aarch32") + SET(TD_LINUX TRUE) + SET(TD_LINUX_32 FALSE) + SET(TD_ARM_32 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + SET(CPUTYPE "aarch64") + MESSAGE(STATUS "Set CPUTYPE to aarch64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_ARM_64 TRUE) + ENDIF () + ELSE () - MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) + # if generate ARM version: + # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 + IF (${CPUTYPE} MATCHES "aarch32") + SET(TD_LINUX TRUE) + SET(TD_LINUX_32 FALSE) + SET(TD_ARM_32 TRUE) + MESSAGE(STATUS "input cpuType: aarch32") + ELSEIF (${CPUTYPE} MATCHES "aarch64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_ARM_64 TRUE) + MESSAGE(STATUS "input cpuType: aarch64") + ELSEIF (${CPUTYPE} MATCHES "mips64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_MIPS_64 TRUE) + MESSAGE(STATUS "input cpuType: mips64") + ELSEIF (${CPUTYPE} MATCHES "x64") + MESSAGE(STATUS "input cpuType: x64") + ELSEIF (${CPUTYPE} MATCHES "x86") + MESSAGE(STATUS "input cpuType: x86") + ELSE () + MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) + ENDIF () + ENDIF () # cmake -DOSTYPE=Ningsi diff --git a/cmake/version.inc b/cmake/version.inc old mode 100644 new mode 100755 index 2f0ec81aea2dcc932845537bcb37f5ab64d9bcb8..962f1f60406f3657a379ee99427e110985988c44 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.8.0") + SET(TD_VER_NUMBER "2.0.16.0") ENDIF () IF (DEFINED VERCOMPATIBLE) @@ -13,16 +13,40 @@ ELSE () SET(TD_VER_COMPATIBLE "2.0.0.0") ENDIF () +find_program(HAVE_GIT NAMES git) + IF (DEFINED GITINFO) SET(TD_VER_GIT ${GITINFO}) +ELSEIF (HAVE_GIT) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID) + message(STATUS "git log result:${GIT_COMMITID}") + IF (GIT_COMMITID) + string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) + SET(TD_VER_GIT ${GIT_COMMITID}) + ELSE () + message(STATUS "not a git repository") + SET(TD_VER_GIT "no git commit id") + ENDIF () ELSE () - SET(TD_VER_GIT "community") + message(STATUS "no git cmd") + SET(TD_VER_GIT "no git commit id") ENDIF () IF (DEFINED GITINFOI) SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSEIF (HAVE_GIT) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID) + message(STATUS "git log result:${GIT_COMMITID}") + IF (GIT_COMMITID) + string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) + SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID}) + ELSE () + message(STATUS "not a git repository") + SET(TD_VER_GIT "no git commit id") + ENDIF () ELSE () - SET(TD_VER_GIT_INTERNAL "internal") + message(STATUS "no git cmd") + SET(TD_VER_GIT_INTERNAL "no git commit id") ENDIF () IF (DEFINED VERDATE) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 8c6c73c4400da973726109aa66a50c97e3a2a6f6..cfc17442f5c21c2d002ba42c45ce523c80eb957f 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -9,7 +9,12 @@ ADD_SUBDIRECTORY(lz4) ADD_SUBDIRECTORY(cJson) ADD_SUBDIRECTORY(wepoll) ADD_SUBDIRECTORY(MsvcLibX) +ADD_SUBDIRECTORY(rmonotonic) IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(MQTT-C) -ENDIF () \ No newline at end of file +ENDIF () + +IF (TD_DARWIN AND TD_MQTT) + ADD_SUBDIRECTORY(MQTT-C) +ENDIF () diff --git a/deps/MsvcLibX/include/msvcTime.h b/deps/MsvcLibX/include/msvcTime.h index 1897da58570cf503b05502d0a15c899f1f126015..6f8b5dac8f88e4c3dd40445f1c5512ba2e6e796f 100644 --- a/deps/MsvcLibX/include/msvcTime.h +++ b/deps/MsvcLibX/include/msvcTime.h @@ -38,6 +38,7 @@ typedef int clockid_t; /* Supported values for clockid_t */ #define CLOCK_REALTIME 0 +#define CLOCK_MONOTONIC 1 int clock_gettime(clockid_t clock_id, struct timespec *tp); diff --git a/deps/MsvcLibX/include/msvcUnistd.h b/deps/MsvcLibX/include/msvcUnistd.h index 9ad60625e07be8e9b749d1951375a3a501832366..9b59bae7f026d1daff65641f665f1df4ca6eb3b5 100644 --- a/deps/MsvcLibX/include/msvcUnistd.h +++ b/deps/MsvcLibX/include/msvcUnistd.h @@ -89,11 +89,12 @@ pid_t getppid(void); /* Get parent PID */ /* Path management */ #if defined(_WIN32) -#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) #define realpath realpathU +#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) +// #define realpath realpathU #define CompactPath CompactPathU #else /* _ANSI_SOURCE */ -#define realpath realpathA +// #define realpath realpathA #define CompactPath CompactPathA #endif #endif /* defined(_WIN32) */ diff --git a/deps/MsvcLibX/src/clock_gettime.c b/deps/MsvcLibX/src/clock_gettime.c index fee7532e2e552ec08b615e61e3f4d79276ea6cfc..cbddf7107ae3733186c014c26a769f3f9d386d0e 100644 --- a/deps/MsvcLibX/src/clock_gettime.c +++ b/deps/MsvcLibX/src/clock_gettime.c @@ -34,15 +34,56 @@ #include "msvcTime.h" #include "sys/msvcStat.h" /* For MsvcLibX's Filetime2Timespec */ -int clock_gettime(clockid_t clock_id, struct timespec *pTS) { - FILETIME ft; - if (clock_id != CLOCK_REALTIME) { - errno = EINVAL; - return -1; +#define MS_PER_SEC 1000ULL // MS = milliseconds +#define US_PER_MS 1000ULL // US = microseconds +#define HNS_PER_US 10ULL // HNS = hundred-nanoseconds (e.g., 1 hns = 100 ns) +#define NS_PER_US 1000ULL + +#define HNS_PER_SEC (MS_PER_SEC * US_PER_MS * HNS_PER_US) +#define NS_PER_HNS (100ULL) // NS = nanoseconds +#define NS_PER_SEC (MS_PER_SEC * US_PER_MS * NS_PER_US) + +int clock_gettime_monotonic(struct timespec *tv) { + static LARGE_INTEGER ticksPerSec; + LARGE_INTEGER ticks; + double seconds; + + if (!ticksPerSec.QuadPart) { + QueryPerformanceFrequency(&ticksPerSec); + if (!ticksPerSec.QuadPart) { + errno = ENOTSUP; + return -1; + } } + + QueryPerformanceCounter(&ticks); + + seconds = (double) ticks.QuadPart / (double) ticksPerSec.QuadPart; + tv->tv_sec = (time_t)seconds; + tv->tv_nsec = (long)((ULONGLONG)(seconds * NS_PER_SEC) % NS_PER_SEC); + + return 0; +} + +int clock_gettime_realtime(struct timespec *pTS) { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); Filetime2Timespec(&ft, pTS); + return 0; } +int clock_gettime(clockid_t clock_id, struct timespec *pTS) { + if (clock_id == CLOCK_MONOTONIC) { + return clock_gettime_monotonic(pTS); + } else if (clock_id == CLOCK_REALTIME) { + return clock_gettime_realtime(pTS); + } + + errno = ENOTSUP; + + return -1; +} + #endif /* defined(_WIN32) */ diff --git a/deps/rmonotonic/CMakeLists.txt b/deps/rmonotonic/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b870b7d6d7a3817c38bfd4c488f75890fd115b7f --- /dev/null +++ b/deps/rmonotonic/CMakeLists.txt @@ -0,0 +1,11 @@ +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST) + +add_definitions(-DUSE_PROCESSOR_CLOCK) + +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../MsvcLibX/include) + +ADD_LIBRARY(rmonotonic ${SOURCE_LIST}) +TARGET_INCLUDE_DIRECTORIES(rmonotonic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc) +IF (TD_WINDOWS) + TARGET_LINK_LIBRARIES(rmonotonic MsvcLibXw) +ENDIF () diff --git a/deps/rmonotonic/inc/monotonic.h b/deps/rmonotonic/inc/monotonic.h new file mode 100644 index 0000000000000000000000000000000000000000..0850548d36629b0b1bfba04e818937e275cac5bf --- /dev/null +++ b/deps/rmonotonic/inc/monotonic.h @@ -0,0 +1,56 @@ +#ifndef __MONOTONIC_H +#define __MONOTONIC_H +/* The monotonic clock is an always increasing clock source. It is unrelated to + * the actual time of day and should only be used for relative timings. The + * monotonic clock is also not guaranteed to be chronologically precise; there + * may be slight skew/shift from a precise clock. + * + * Depending on system architecture, the monotonic time may be able to be + * retrieved much faster than a normal clock source by using an instruction + * counter on the CPU. On x86 architectures (for example), the RDTSC + * instruction is a very fast clock source for this purpose. + */ + +//#include "fmacros.h" +#include +//#include + +#if defined(_WIN32) || defined(_WIN64) + #define inline +#endif + +/* A counter in micro-seconds. The 'monotime' type is provided for variables + * holding a monotonic time. This will help distinguish & document that the + * variable is associated with the monotonic clock and should not be confused + * with other types of time.*/ +typedef uint64_t monotime; + +/* Retrieve counter of micro-seconds relative to an arbitrary point in time. */ +extern monotime (*getMonotonicUs)(void); + + +/* Call once at startup to initialize the monotonic clock. Though this only + * needs to be called once, it may be called additional times without impact. + * Returns a printable string indicating the type of clock initialized. + * (The returned string is static and doesn't need to be freed.) */ +const char * monotonicInit(); + + +/* Functions to measure elapsed time. Example: + * monotime myTimer; + * elapsedStart(&myTimer); + * while (elapsedMs(myTimer) < 10) {} // loops for 10ms + */ +static inline void elapsedStart(monotime *start_time) { + *start_time = getMonotonicUs(); +} + +static inline uint64_t elapsedUs(monotime start_time) { + return getMonotonicUs() - start_time; +} + +static inline uint64_t elapsedMs(monotime start_time) { + return elapsedUs(start_time) / 1000; +} + +#endif diff --git a/deps/rmonotonic/src/monotonic.c b/deps/rmonotonic/src/monotonic.c new file mode 100644 index 0000000000000000000000000000000000000000..1470f91b56c79b4ee2d8429ecf58fc365d03e737 --- /dev/null +++ b/deps/rmonotonic/src/monotonic.c @@ -0,0 +1,173 @@ +#include "monotonic.h" +#include +#include +#include +#include + +#undef NDEBUG +#include + +#if defined(_WIN32) || defined(_WIN64) +#include "msvcTime.h" +#include "msvcStdio.h" +#endif + +/* The function pointer for clock retrieval. */ +monotime (*getMonotonicUs)(void) = NULL; + +static char monotonic_info_string[32]; + + +/* Using the processor clock (aka TSC on x86) can provide improved performance + * throughout Redis wherever the monotonic clock is used. The processor clock + * is significantly faster than calling 'clock_getting' (POSIX). While this is + * generally safe on modern systems, this link provides additional information + * about use of the x86 TSC: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage + * + * To use the processor clock, either uncomment this line, or build with + * CFLAGS="-DUSE_PROCESSOR_CLOCK" +#define USE_PROCESSOR_CLOCK + */ + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) +#include +#include + +static long mono_ticksPerMicrosecond = 0; + +static monotime getMonotonicUs_x86() { + return __rdtsc() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_x86linux() { + const int bufflen = 256; + char buf[bufflen]; + regex_t cpuGhzRegex, constTscRegex; + const size_t nmatch = 2; + regmatch_t pmatch[nmatch]; + int constantTsc = 0; + int rc; + + /* Determine the number of TSC ticks in a micro-second. This is + * a constant value matching the standard speed of the processor. + * On modern processors, this speed remains constant even though + * the actual clock speed varies dynamically for each core. */ + rc = regcomp(&cpuGhzRegex, "^model name\\s+:.*@ ([0-9.]+)GHz", REG_EXTENDED); + assert(rc == 0); + + /* Also check that the constant_tsc flag is present. (It should be + * unless this is a really old CPU. */ + rc = regcomp(&constTscRegex, "^flags\\s+:.* constant_tsc", REG_EXTENDED); + assert(rc == 0); + + FILE *cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo != NULL) { + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&cpuGhzRegex, buf, nmatch, pmatch, 0) == 0) { + buf[pmatch[1].rm_eo] = '\0'; + double ghz = atof(&buf[pmatch[1].rm_so]); + mono_ticksPerMicrosecond = (long)(ghz * 1000); + break; + } + } + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&constTscRegex, buf, nmatch, pmatch, 0) == 0) { + constantTsc = 1; + break; + } + } + + fclose(cpuinfo); + } + regfree(&cpuGhzRegex); + regfree(&constTscRegex); + + if (mono_ticksPerMicrosecond == 0) { + //fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate"); + return; + } + if (!constantTsc) { + //fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "X86 TSC @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_x86; +} +#endif + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__) +static long mono_ticksPerMicrosecond = 0; + +/* Read the clock value. */ +static inline uint64_t __cntvct() { + uint64_t virtual_timer_value; + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +} + +/* Read the Count-timer Frequency. */ +static inline uint32_t cntfrq_hz() { + uint64_t virtual_freq_value; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(virtual_freq_value)); + return (uint32_t)virtual_freq_value; /* top 32 bits are reserved */ +} + +static monotime getMonotonicUs_aarch64() { + return __cntvct() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_aarch64() { + mono_ticksPerMicrosecond = (long)cntfrq_hz() / 1000L / 1000L; + if (mono_ticksPerMicrosecond == 0) { + fprintf(stderr, "monotonic: aarch64, unable to determine clock rate"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "ARM CNTVCT @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_aarch64; +} +#endif + +static monotime getMonotonicUs_posix(void) { + /* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems + * did not support this until much later. CLOCK_MONOTONIC is technically + * optional and may not be supported - but it appears to be universal. + * If this is not supported, provide a system-specific alternate version. */ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000; +} + +static void monotonicInit_posix() { + /* Ensure that CLOCK_MONOTONIC is supported. This should be supported + * on any reasonably current OS. If the assertion below fails, provide + * an appropriate alternate implementation. */ + struct timespec ts; + int rc = clock_gettime(CLOCK_MONOTONIC, &ts); + assert(rc == 0); + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "POSIX clock_gettime"); + getMonotonicUs = getMonotonicUs_posix; +} + + + +const char * monotonicInit() { + #if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) + if (getMonotonicUs == NULL) monotonicInit_x86linux(); + #endif + + #if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__) + if (getMonotonicUs == NULL) monotonicInit_aarch64(); + #endif + + if (getMonotonicUs == NULL) monotonicInit_posix(); + + return monotonic_info_string; +} diff --git a/deps/zlib-1.2.11/src/gzlib.c b/deps/zlib-1.2.11/src/gzlib.c index 381cdc2c7debea386cc4d70114d6b8823a521d16..4c135aabffffa7aa13d96f19b6bd2f059d7cec6e 100644 --- a/deps/zlib-1.2.11/src/gzlib.c +++ b/deps/zlib-1.2.11/src/gzlib.c @@ -5,6 +5,10 @@ #include #include "gzguts.h" +#ifndef O_BINARY + #define O_BINARY 0 +#endif + #if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) # define LSEEK _lseeki64 #else @@ -240,9 +244,9 @@ local gzFile gz_open(path, fd, mode) /* open the file with the appropriate flags (or just use fd) */ state->fd = fd > -1 ? fd : ( #ifdef WIDECHAR - fd == -2 ? _wopen(path, oflag, 0666) : + fd == -2 ? _wopen(path, oflag | O_BINARY, 0666) : #endif - open((const char *)path, oflag, 0666)); + open((const char *)path, oflag | O_BINARY, 0666)); if (state->fd == -1) { free(state->path); free(state); diff --git a/documentation/tdenginedocs-en/faq/index.html b/documentation/tdenginedocs-en/faq/index.html index a2fb8ba68ba909c2ad8273e6c5b8aa9e4a1d109b..2d716cd6dc1b4bc233dbe3a680fa17286f651c91 100644 --- a/documentation/tdenginedocs-en/faq/index.html +++ b/documentation/tdenginedocs-en/faq/index.html @@ -1,5 +1,5 @@ Documentation | Taos Data
Back

FAQ

-

1. When encoutered with the error "failed to connect to server", what can I do?

+

1. When encountered with the error "failed to connect to server", what can I do?

The client may encounter connection errors. Please follow the steps below for troubleshooting:

  1. On the server side, execute systemctl status taosd to check the status of taosd service. If taosd is not running, start it and retry connecting.
    2. diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..c16673cba59d42ab00aa2975cfa2341ede9e5fbb --- /dev/null +++ b/documentation20/cn/00.index/docs.md @@ -0,0 +1,153 @@ +# TDengine文档 + +TDengine是一个高效的存储、查询、分析时序大数据的平台，专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它，但建议您在使用前仔细阅读一遍下面的文档，特别是 [数据模型](/architecture) 与 [数据建模](/model)。除本文档之外，欢迎 [下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。如需查阅TDengine 1.6 文档，请点击 [这里](https://www.taosdata.com/cn/documentation16/) 访问。 + +## [TDengine介绍](/evaluation) + +* [TDengine 简介及特色](/evaluation#intro) +* [TDengine 适用场景](/evaluation#scenes) +* [TDengine 性能指标介绍和验证方法](/evaluation#) + +## [立即开始](/getting-started) + +* [快捷安装](/getting-started#install)：可通过源码、安装包或docker安装，三秒钟搞定 +* [轻松启动](/getting-started#start)：使用systemctl 启停TDengine +* [命令行程序TAOS](/getting-started#console)：访问TDengine的简便方式 +* [极速体验](/getting-started#demo)：运行示例程序，快速体验高效的数据插入、查询 +* [支持平台列表](/getting-started#platforms)：TDengine服务器和客户端支持的平台列表 + +## [整体架构](/architecture) + +* [数据模型](/architecture#model)：关系型数据库模型，但要求每个采集点单独建表 +* [集群与基本逻辑单元](/architecture#cluster)：吸取NoSQL优点，支持水平扩展，支持高可靠 +* [存储模型与数据分区、分片](/architecture#sharding)：标签数据与时序数据完全分离，按vnode和时间两个维度对数据切分 +* [数据写入与复制流程](/architecture#replication)：先写入WAL、之后写入缓存，再给应用确认，支持多副本 +* [缓存与持久化](/architecture#persistence)：最新数据缓存在内存中，但落盘时采用列式存储、超高压缩比 +* [数据查询](/architecture#query)：支持各种函数、时间轴聚合、插值、多表聚合 + +## [数据建模](/model) + +* [创建库](/model#create-db)：为具有相似数据特征的数据采集点创建一个库 +* [创建超级表](/model#create-stable)：为同一类型的数据采集点创建一个超级表 +* [创建表](/model#create-table)：使用超级表做模板，为每一个具体的数据采集点单独建表 + +## [TAOS SQL](/taos-sql) + +* [支持的数据类型](/taos-sql#data-type)：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 +* [数据库管理](/taos-sql#management)：添加、删除、查看数据库 +* [表管理](/taos-sql#table)：添加、删除、查看、修改表 +* [超级表管理](/taos-sql#super-table)：添加、删除、查看、修改超级表 +* [标签管理](/taos-sql#tags)：增加、删除、修改标签 +* [数据写入](/taos-sql#insert)：支持单表单条、多条、多表多条写入，支持历史数据写入 +* [数据查询](/taos-sql#select)：支持时间段、值过滤、排序、查询结果手动分页等 +* [SQL函数](/taos-sql#functions)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等 +* [时间维度聚合](/taos-sql#aggregation)：将表中数据按照时间段进行切割后聚合，降维处理 +* [边界限制](/taos-sql#limitation)：库、表、SQL等边界限制条件 +* [错误码](/taos-sql/error-code)：TDengine 2.0 错误码以及对应的十进制码 + +## [高效写入数据](/insert) + +* [SQL写入](/insert#sql)：使用SQL insert命令向一张或多张表写入单条或多条记录 +* [Prometheus写入](/insert#prometheus)：配置Prometheus, 不用任何代码，将数据直接写入 +* [Telegraf写入](/insert#telegraf)：配置Telegraf, 不用任何代码，将采集数据直接写入 +* [EMQ X Broker](/insert#emq)：配置EMQ X，不用任何代码，就可将MQTT数据直接写入 +* [HiveMQ Broker](/insert#hivemq)：配置HiveMQ，不用任何代码，就可将MQTT数据直接写入 + +## [高效查询数据](/queries) + +* [主要查询功能](/queries#queries)：支持各种标准函数，设置过滤条件，时间段查询 +* [多表聚合查询](/queries#aggregation)：使用超级表，设置标签过滤条件，进行高效聚合查询 +* [降采样查询值](/queries#sampling)：按时间段分段聚合，支持插值 + +## [高级功能](/advanced-features) + +* [连续查询(Continuous Query)](/advanced-features#continuous-query)：基于滑动窗口，定时自动的对数据流进行查询计算 +* [数据订阅(Publisher/Subscriber)](/advanced-features#subscribe)：象典型的消息队列，应用可订阅接收到的最新数据 +* [缓存(Cache)](/advanced-features#cache)：每个设备最新的数据都会缓存在内存中，可快速获取 +* [报警监测](/advanced-features#alert)：根据配置规则，自动监测超限行为数据，并主动推送 + +## [连接器](/connector) + +* [C/C++ Connector](/connector#c-cpp)：通过libtaos客户端的库，连接TDengine服务器的主要方法 +* [Java Connector(JDBC)](/connector/java)：通过标准的JDBC API，给Java应用提供到TDengine的连接 +* [Python Connector](/connector#python)：给Python应用提供一个连接TDengine服务器的驱动 +* [RESTful Connector](/connector#restful)：提供一最简单的连接TDengine服务器的方式 +* [Go Connector](/connector#go)：给Go应用提供一个连接TDengine服务器的驱动 +* [Node.js Connector](/connector#nodejs)：给node应用提供一个连接TDengine服务器的驱动 +* [C# Connector](/connector#csharp)：给C#应用提供一个连接TDengine服务器的驱动 +* [Windows客户端](https://www.taosdata.com/blog/2019/07/26/514.html)：自行编译windows客户端，Windows环境的各种连接器都需要它 + +## [与其他工具的连接](/connections) + +* [Grafana](/connections#grafana)：获取并可视化保存在TDengine的数据 +* [Matlab](/connections#matlab)：通过配置Matlab的JDBC数据源访问保存在TDengine的数据 +* [R](/connections#r)：通过配置R的JDBC数据源访问保存在TDengine的数据 +* [IDEA Database](https://www.taosdata.com/blog/2020/08/27/1767.html)：通过IDEA 数据库管理工具可视化使用 TDengine + +## [TDengine集群的安装、管理](/cluster) + +* [准备工作](/cluster#prepare)：部署环境前的几点注意事项 +* [创建第一个节点](/cluster#node-one)：与快捷安装完全一样，非常简单 +* [创建后续节点](/cluster#node-other)：配置新节点的taos.cfg, 在现有集群添加新的节点 +* [节点管理](/cluster#management)：增加、删除、查看集群的节点 +* [Vnode 的高可用性](/cluster#high-availability)：通过多副本的机制来提供 Vnode 的高可用性 +* [Mnode 的管理](/cluster#mnode)：系统自动创建、无需任何人工干预 +* [负载均衡](/cluster#load-balancing)：一旦节点个数或负载有变化，自动进行 +* [节点离线处理](/cluster#offline)：节点离线超过一定时长，将从集群中剔除 +* [Arbitrator](/cluster#arbitrator)：对于偶数个副本的情形，使用它可以防止split brain + +## [TDengine的运营和维护](/administrator) + +* [容量规划](/administrator#planning)：根据场景，估算硬件资源 +* [容错和灾备](/administrator#tolerance)：设置正确的WAL和数据副本数 +* [系统配置](/administrator#config)：端口，缓存大小，文件块大小和其他系统配置 +* [用户管理](/administrator#user)：添加、删除TDengine用户，修改用户密码 +* [数据导入](/administrator#import)：可按脚本文件导入，也可按数据文件导入 +* [数据导出](/administrator#export)：从shell按表导出，也可用taosdump工具做各种导出 +* [系统监控](/administrator#status)：检查系统现有的连接、查询、流式计算，日志和事件等 +* [文件目录结构](/administrator#directories)：TDengine数据文件、配置文件等所在目录 +* [参数限制与保留关键字](/administrator#keywords)：TDengine的参数限制与保留关键字列表 + +## TDengine的技术设计 + +* [系统模块](/architecture/taosd)：taosd的功能和模块划分 +* [数据复制](/architecture/replica)：支持实时同步、异步复制，保证系统的High Availibility +* [技术博客](https://www.taosdata.com/cn/blog/?categories=3)：更多的技术分析和架构设计文章 + +## 常用工具 + +* [TDengine样例导入工具](https://www.taosdata.com/blog/2020/01/18/1166.html) +* [TDengine性能对比测试工具](https://www.taosdata.com/blog/2020/01/18/1166.html) +* [IDEA数据库管理工具可视化使用TDengine](https://www.taosdata.com/blog/2020/08/27/1767.html) +* [基于eletron开发的跨平台TDengine图形化管理工具](https://github.com/skye0207/TDengineGUI) +* [DataX，支持TDengine的离线数据采集/同步工具](https://github.com/alibaba/DataX) + +## TDengine与其他数据库的对比测试 + +* [用InfluxDB开源的性能测试工具对比InfluxDB和TDengine](https://www.taosdata.com/blog/2020/01/13/1105.html) +* [TDengine与OpenTSDB对比测试](https://www.taosdata.com/blog/2019/08/21/621.html) +* [TDengine与Cassandra对比测试](https://www.taosdata.com/blog/2019/08/14/573.html) +* [TDengine与InfluxDB对比测试](https://www.taosdata.com/blog/2019/07/19/419.html) +* [TDengine与InfluxDB、OpenTSDB、Cassandra、MySQL、ClickHouse等数据库的对比测试报告](https://www.taosdata.com/downloads/TDengine_Testing_Report_cn.pdf) + +## 物联网大数据 + +* [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/105.html) +* [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/542.html) +* [通用大数据架构为什么不适合处理物联网数据？](https://www.taosdata.com/blog/2019/07/09/107.html) +* [物联网、车联网、工业互联网大数据平台，为什么推荐使用TDengine？](https://www.taosdata.com/blog/2019/07/09/109.html) + +## 培训和FAQ + +* [FAQ：常见问题与答案](/faq) +* [技术公开课：开源、高效的物联网大数据平台，TDengine内核技术剖析](https://www.taosdata.com/blog/2020/12/25/2126.html) +* [TDengine视频教程-快速上手](https://www.taosdata.com/blog/2020/11/11/1941.html) +* [TDengine视频教程-数据建模](https://www.taosdata.com/blog/2020/11/11/1945.html) +* [TDengine视频教程-集群搭建](https://www.taosdata.com/blog/2020/11/11/1961.html) +* [TDengine视频教程-Go Connector](https://www.taosdata.com/blog/2020/11/11/1951.html) +* [TDengine视频教程-JDBC Connector](https://www.taosdata.com/blog/2020/11/11/1955.html) +* [TDengine视频教程-NodeJS Connector](https://www.taosdata.com/blog/2020/11/11/1957.html) +* [TDengine视频教程-Python Connector](https://www.taosdata.com/blog/2020/11/11/1963.html) +* [TDengine视频教程-RESTful Connector](https://www.taosdata.com/blog/2020/11/11/1965.html) +* [TDengine视频教程-“零”代码运维监控](https://www.taosdata.com/blog/2020/11/11/1959.html) +* [应用案例：一些使用实例来解释如何使用TDengine](https://www.taosdata.com/cn/blog/?categories=4) diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/cn/01.evaluation/docs.md similarity index 97% rename from documentation20/webdocs/markdowndocs/Evaluation-ch.md rename to documentation20/cn/01.evaluation/docs.md index a92f97a8d9dfc9a47d5554daa076f4d2f0774c92..0ae2106ff2a63696dc8bbc51d25bbf5e811ef561 100644 --- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -1,6 +1,6 @@ # TDengine 介绍 -## TDengine 简介 +## TDengine 简介 TDengine是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品，它不依赖任何第三方软件，也不是优化或包装了一个开源的数据库或流式计算产品，而是在吸取众多传统关系型数据库、NoSQL数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品，在时序空间大数据处理上，有着自己独到的优势。 @@ -15,10 +15,11 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 -
    +![TDengine技术生态图](page://images/eco_system.png)
    图 1. TDengine技术生态图
    -## TDengine 总体适用场景 + +## TDengine 总体适用场景 作为一个IOT大数据平台，TDengine的典型适用场景是在IOT范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如CRM，ERP等，不在本文讨论范围内。 @@ -59,5 +60,3 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 |要求运维学习成本可控| | | √ |同上。| |要求市场有大量人才储备| √ | | |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务。| -## TDengine 性能指标介绍和验证方法 - diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..c9c82942f3bd47fa774636dd5cf7c5b33f879209 --- /dev/null +++ b/documentation20/cn/02.getting-started/docs.md @@ -0,0 +1,220 @@ +# 立即开始 + +## 快捷安装 + +TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 + +### 通过源码安装 + +请参考我们的[TDengine github主页](https://github.com/taosdata/TDengine)下载源码并安装. + +### 通过Docker容器运行 + +请参考[TDengine官方Docker镜像的发布、下载和使用](https://www.taosdata.com/blog/2020/05/13/1509.html) + +### 通过安装包安装 + +TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择： + +安装包下载在[这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。 + +具体的安装过程，请参见[TDengine多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html)以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。 + +## 轻松启动 + +安装成功后，用户可使用`systemctl`命令来启动TDengine的服务进程。 + +```bash +$ systemctl start taosd +``` + +检查服务是否正常工作。 +```bash +$ systemctl status taosd +``` + +如果TDengine服务正常工作，那么您可以通过TDengine的命令行程序`taos`来访问并体验TDengine。 + +**注意：** + +- systemctl命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo +- 为更好的获得产品反馈，改善产品，TDengine会采集基本的使用信息，但您可以修改系统配置文件taos.cfg里的配置参数telemetryReporting, 将其设为0，就可将其关闭。 +- TDengine采用FQDN(一般就是hostname)作为节点的ID，为保证正常运行，需要给运行taosd的服务器配置好hostname，在客户端应用运行的机器配置好DNS服务或hosts文件，保证FQDN能够解析。 + +* TDengine 支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装，用`which systemctl`命令来检测系统中是否存在`systemd`包: + + ```bash + $ which systemctl + ``` + + 如果系统中不支持systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 + + +## TDengine命令行程序 + +执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。 + +```bash +$ taos +``` + +如果TDengine终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/documentation/faq/)来解决终端连接服务端失败的问题）。TDengine终端的提示符号如下： + +```cmd +taos> +``` + +在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例： + +```mysql +create database demo; +use demo; +create table t (ts timestamp, speed int); +insert into t values ('2019-07-15 00:00:00', 10); +insert into t values ('2019-07-15 01:00:00', 20); +select * from t; + ts | speed | +=================================== + 19-07-15 00:00:00.000| 10| + 19-07-15 01:00:00.000| 20| +Query OK, 2 row(s) in set (0.001700s) +``` + +除执行SQL语句外，系统管理员还可以从TDengine终端检查系统运行状态，添加删除用户账号等。 + +### 命令行参数 + +您可通过配置命令行参数来改变TDengine终端的行为。以下为常用的几个命令行参数： + +- -c, --config-dir: 指定配置文件目录，默认为_/etc/taos_ +- -h, --host: 指定服务的IP地址，默认为本地服务 +- -s, --commands: 在不进入终端的情况下运行TDengine命令 +- -u, -- user: 连接TDengine服务器的用户名，缺省为root +- -p, --password: 连接TDengine服务器的密码，缺省为taosdata +- -?, --help: 打印出所有命令行参数 + +示例： + +```bash +$ taos -h 192.168.0.1 -s "use db; show tables;" +``` + +### 运行SQL命令脚本 + +TDengine终端可以通过`source`命令来运行SQL命令脚本. + +```mysql +taos> source ; +``` + +### Shell小技巧 + +- 可以使用上下光标键查看已经历史输入的命令 +- 修改用户密码。在shell中使用alter user命令 +- ctrl+c 中止正在进行中的查询 +- 执行`RESET QUERY CACHE`清空本地缓存的表的schema + + +## TDengine 极速体验 + +启动TDengine的服务，在Linux终端执行taosdemo + +```bash +$ taosdemo +``` + +该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。 + +执行这条命令大概需要10分钟，最后共插入10亿条记录。 + +在TDengine客户端输入查询命令，体验查询速度。 + +- 查询超级表下记录总条数： + +```mysql +taos> select count(*) from test.meters; +``` + +- 查询10亿条记录的平均值、最大值、最小值等： + +```mysql +taos> select avg(f1), max(f2), min(f3) from test.meters; +``` + +- 查询loc="beijing"的记录总条数： + +```mysql +taos> select count(*) from test.meters where loc="beijing"; +``` + +- 查询areaid=10的所有记录的平均值、最大值、最小值等： + +```mysql +taos> select avg(f1), max(f2), min(f3) from test.meters where areaid=10; +``` + +- 对表t10按10s进行平均值、最大值和最小值聚合统计： + +```mysql +taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); +``` + +**Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。 + + +## 客户端和报警模块 + +如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux和Windows安装包如下： + +- TDengine-client-2.0.10.0-Linux-x64.tar.gz(3.0M) +- TDengine-client-2.0.10.0-Windows-x64.exe(2.8M) +- TDengine-client-2.0.10.0-Windows-x86.exe(2.8M) + +报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）： + +- TDengine-alert-2.0.10.0-Linux-x64.tar.gz (8.1M) + + +## 支持平台列表 + +### TDengine服务器支持的平台列表 + +| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** | +| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | +| X64 | ● | ● | | ○ | ● | ● | +| 树莓派ARM32 | | ● | ● | | | | +| 龙芯MIPS64 | | | ● | | | | +| 鲲鹏 ARM64 | | ○ | ○ | | ● | | +| 申威 Alpha64 | | | ○ | ● | | | +| 飞腾ARM64 | | ○优麒麟 | | | | | +| 海光X64 | ● | ● | ● | ○ | ● | ● | +| 瑞芯微ARM64/32 | | | ○ | | | | +| 全志ARM64/32 | | | ○ | | | | +| 炬力ARM64/32 | | | ○ | | | | +| TI ARM32 | | | ○ | | | | + +注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 + + + +### TDengine客户端和连接器支持的平台列表 + +目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。 + +对照矩阵如下： + +| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** | +| ----------- | --------------- | --------- | --------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ | +| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | +| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | +| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | +| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● | +| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | +| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | +| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- | +| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● | + +注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 + +请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。 + diff --git a/documentation20/webdocs/markdowndocs/taosd-ch.md b/documentation20/cn/03.architecture/01.taosd/docs.md similarity index 97% rename from documentation20/webdocs/markdowndocs/taosd-ch.md rename to documentation20/cn/03.architecture/01.taosd/docs.md index 08be0c163e7076b58f03ff8ea3165e902a80fe64..66d51ed2dc2ea1546ab167cad680c20b3fa9729c 100644 --- a/documentation20/webdocs/markdowndocs/taosd-ch.md +++ b/documentation20/cn/03.architecture/01.taosd/docs.md @@ -3,9 +3,10 @@ 逻辑上，TDengine系统包含dnode, taosc和App，dnode是服务器侧执行代码taosd的一个运行实例，因此taosd是TDengine的核心，本文对taosd的设计做一简单的介绍，模块内的实现细节请见其他文档。 ## 系统模块图 + taosd包含rpc, dnode, vnode, tsdb, query, cq, sync, wal, mnode, http, monitor等模块，具体如下图： -
    +![modules.png](page://images/architecture/modules.png) taosd的启动入口是dnode模块，dnode然后启动其他模块，包括可选配置的http, monitor模块。taosc或dnode之间交互的消息都是通过rpc模块进行，dnode模块根据接收到的消息类型，将消息分发到vnode或mnode的消息队列，或由dnode模块自己消费。dnode的工作线程(worker)消费消息队列里的消息，交给mnode或vnode进行处理。下面对各个模块做简要说明。 @@ -40,13 +41,14 @@ RPC模块还提供数据压缩功能，如果数据包的字节数超过系统 taosd的消息消费由dnode通过读写线程池进行控制，是系统的中枢。该模块内的结构体图如下： -
    +![dnode.png](page://images/architecture/dnode.png) ## VNODE模块 vnode是一独立的数据存储查询逻辑单元，但因为一个vnode只能容许一个DB，因此vnode内部没有account, DB, user等概念。为实现更好的模块化、封装以及未来的扩展，它有很多子模块，包括负责存储的TSDB，负责查询的Query, 负责数据复制的sync，负责数据库日志的的wal, 负责连续查询的cq(continuous query), 负责事件触发的流计算的event等模块，这些子模块只与vnode模块发生关系，与其他模块没有任何调用关系。模块图如下： -
    +![vnode.png](page://images/architecture/vnode.png) + vnode模块向下，与dnodeVRead，dnodeVWrite发生互动，向上，与子模块发生互动。它主要的功能有： - 协调各个子模块的互动。各个子模块之间都不直接调用，都需要通过vnode模块进行； @@ -68,30 +70,37 @@ mnode是整个系统的大脑，负责整个系统的资源调度，负责meta d mnode里还负责account, user, DB, stable, table, vgroup, dnode的创建、删除与更新。mnode不仅把这些entity的meta data保存在内存，还做持久化存储。但为节省内存，各个表的标签值不保存在mnode（保存在vnode)，而且子表不维护自己的schema, 而是与stable共享。为减小mnode的查询压力，taosc会缓存table、stable的schema。对于查询类的操作，各个slave mnode也可以提供，以减轻master压力。 ## TSDB模块 + TSDB模块是VNODE中的负责快速高并发地存储和读取属于该VNODE的表的元数据及采集的时序数据的引擎。除此之外，TSDB还提供了表结构的修改、表标签值的修改等功能。TSDB提供API供VNODE和Query等模块调用。TSDB中存储了两类数据，1：元数据信息；2：时序数据 ### 元数据信息 + TSDB中存储的元数据包含属于其所在的VNODE中表的类型，schema的定义等。对于超级表和超级表下的子表而言，又包含了tag的schema定义以及子表的tag值等。对于元数据信息而言，TSDB就相当于一个全内存的KV型数据库，属于该VNODE的表对象全部在内存中，方便快速查询表的信息。除此之外，TSDB还对其中的子表，按照tag的第一列取值做了全内存的索引，大大加快了对于标签的过滤查询。TSDB中的元数据的最新状态在落盘时，会以追加（append-only）的形式，写入到meta文件中。meta文件只进行追加操作，即便是元数据的删除，也会以一条记录的形式写入到文件末尾。TSDB也提供了对于元数据的修改操作，如表schema的修改，tag schema的修改以及tag值的修改等。 ### 时序数据 + 每个TSDB在创建时，都会事先分配一定量的内存缓冲区，且内存缓冲区的大小可配可修改。表采集的时序数据，在写入TSDB时，首先以追加的方式写入到分配的内存缓冲区中，同时建立基于时间戳的内存索引，方便快速查询。当内存缓冲区的数据积累到一定的程度时（达到内存缓冲区总大小的1/3），则会触发落盘操作，将缓冲区中的数据持久化到硬盘文件上。时序数据在内存缓冲区中是以行（row）的形式存储的。 而时序数据在写入到TSDB的数据文件时，是以列（column）的形式存储的。TSDB中的数据文件包含多个数据文件组，每个数据文件组中又包含.head、.data和.last三个文件，如（v2f1801.head、v2f1801.data、v2f1801.last）数据文件组。TSDB中的数据文件组是按照时间跨度进行分片的，默认是10天一个文件组，且可通过配置文件及建库选项进行配置。分片的数据文件组又按照编号递增排列，方便快速定位某一时间段的时序数据，高效定位数据文件组。时序数据在TSDB的数据文件中是以块的形式进行列式存储的，每个块中只包含一张表的数据，且数据在一个块中是按照时间顺序递增排列的。在一个数据文件组中，.head文件负责存储数据块的索引及统计信息，如每个块的位置，压缩算法，时间戳范围等。存储在.head文件中一张表的索引信息是按照数据块中存储的数据的时间递增排列的，方便进行折半查找等工作。.head和.last文件是存储真实数据块的文件，若数据块中的数据累计到一定程度，则会写入.data文件中，否则，会写入.last文件中，等待下次落盘时合并数据写入.data文件中，从而大大减少文件中块的个数，避免数据的过度碎片化。 ## Query模块 + 该模块负责整体系统的查询处理。客户端调用该该模块进行SQL语法解析，并将查询或写入请求发送到vnode，同时负责针对超级表的查询进行二阶段的聚合操作。在Vnode端，该模块调用TSDB模块读取系统中存储的数据进行查询处理。Query模块还定义了系统能够支持的全部查询函数，查询函数的实现机制与查询框架无耦合，可以在不修改查询流程的情况下动态增加查询函数。详细的设计请参见《TDengine 2.0查询模块设计》。 ## SYNC模块 + 该模块实现数据的多副本复制，包括vnode与mnode的数据复制，支持异步和同步两种复制方式，以满足meta data与时序数据不同复制的需求。因为它为mnode与vnode共享，系统为mnode副本预留了一个特殊的vgroup ID:1。因此vnode group的ID是从2开始的。 -每个vnode/mnode模块实例会有一对应的sync模块实例，他们是一一对应的。详细设计请见TDengine 2.0 数据复制模块设计 +每个vnode/mnode模块实例会有一对应的sync模块实例，他们是一一对应的。详细设计请见[TDengine 2.0 数据复制模块设计](https://www.taosdata.com/cn/documentation/architecture/replica/) ## WAL模块 + 该模块负责将新插入的数据写入write ahead log(WAL), 为vnode, mnode共享。以保证服务器crash或其他故障，能从WAL中恢复数据。 每个vnode/mnode模块实例会有一对应的wal模块实例，是完全一一对应的。WAL的落盘操作由两个参数walLevel, fsync控制。看具体场景，如果要100%保证数据不会丢失，需要将walLevel配置为2，fsync设置为0，每条数据插入请求，都会实时落盘后，才会给应用确认 ## HTTP模块 + 该模块负责处理系统对外的RESTful接口，可以通过配置，由dnode启动或停止。 该模块将接收到的RESTful请求，做了各种合法性检查后，将其变成标准的SQL语句，通过taosc的异步接口，将请求发往整个系统中的任一dnode。收到处理后的结果后，再翻译成HTTP协议，返回给应用。 @@ -99,6 +108,7 @@ TSDB中存储的元数据包含属于其所在的VNODE中表的类型，schema 如果HTTP模块启动，就意味着启动了一个taosc的实例。任一一个dnode都可以启动该模块，以实现对RESTful请求的分布式处理。 ## Monitor模块 + 该模块负责检测一个dnode的运行状态，可以通过配置，由dnode启动或停止。原则上，每个dnode都应该启动一个monitor实例。 Monitor采集TDengine里的关键操作，比如创建、删除、更新账号、表、库等，而且周期性的收集CPU、内存、网络等资源的使用情况（采集周期由系统配置参数monitorInterval控制）。获得这些数据后，monitor模块将采集的数据写入系统的日志库(DB名字由系统配置参数monitorDbName控制）。 diff --git a/documentation20/webdocs/markdowndocs/replica-ch.md b/documentation20/cn/03.architecture/02.replica/docs.md similarity index 93% rename from documentation20/webdocs/markdowndocs/replica-ch.md rename to documentation20/cn/03.architecture/02.replica/docs.md index 1d80174455913a5de55678151811218e6f3d749c..8e1b1e3ab1513fbeaa5b9b805263485a13483b9b 100644 --- a/documentation20/webdocs/markdowndocs/replica-ch.md +++ b/documentation20/cn/03.architecture/02.replica/docs.md @@ -10,13 +10,13 @@ TDengine面向的是物联网场景，需要支持数据的实时复制，来最 数据复制是与数据存储（写入、读取）密切相关的，但两者又是相对独立，可以完全脱耦的。在TDengine系统中，有两种不同类型的数据，一种是时序数据，由TSDB模块负责；一种是元数据(Meta Data), 由MNODE负责。这两种性质不同的数据都需要同步功能。数据复制模块通过不同的实例启动配置参数，为这两种类型数据都提供同步功能。 -在阅读本文之前，请先阅读《TDengine 2.0 整体架构》，了解TDengine的集群设计和基本概念 +在阅读本文之前，请先阅读《[TDengine 2.0 整体架构](https://www.taosdata.com/cn/documentation/architecture/)》，了解TDengine的集群设计和基本概念 特别注明：本文中提到数据更新操作包括数据的增加、删除与修改。 ## 基本概念和定义 -TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储元数据。但从同步数据复制的模块来看，两者没有本质的区别，因此本文里的虚拟节点不仅包括vnode, 也包括mnode, vgoup也指mnode group, 除非特别注明。 +TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储元数据。但从同步数据复制的模块来看，两者没有本质的区别，因此本文里的虚拟节点不仅包括vnode, 也包括mnode, vgroup也指mnode group, 除非特别注明。 **版本(version)**： @@ -30,7 +30,7 @@ TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储 - master： 具有最新的数据，容许客户端往里写入数据，一个虚拟节点组，至多一个master. - slave：与master是同步的，但不容许客户端往里写入数据，根据配置，可以容许客户端对其进行查询。 -- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务 +- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务。 - offline: 由于宕机或网络原因，无法访问到某虚拟节点时，其他虚拟节点将该虚拟节点标为离线。但请注意，该虚拟节点本身的状态可能是unsynced或其他，但不会是离线。 **Quorum:** @@ -83,14 +83,14 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性 如果一个虚拟节点(vnode A)检测到与同一虚拟节点组内另外一虚拟节点（vnode B）的连接中断，vnode A将立即把vnode B的role设置为offline。无论是接收到另外一虚拟节点发来的status消息，还是检测与另外一虚拟节点的连接中断，该虚拟节点都将进入状态处理流程。状态处理流程的规则如下： 1. 如果检测到在线的节点数没有超过一半，则将自己的状态设置为unsynced. -2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程 +2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程。 3. 如果master不存在，则会检查自己保存的各虚拟节点的状态信息与从另一节点接收到的是否一致，如果一致，说明节点组里状态已经稳定一致，则会触发选举流程。如果不一致，说明状态还没趋于一致，即使master不存在，也不进行选主。由于要求状态信息一致才进行选举，每个虚拟节点根据同样的信息，会选出同一个虚拟节点做master，无需投票表决。 4. 自己的状态是根据规则自己决定并修改的，并不需要其他节点同意，包括成为master。一个节点无权修改其他节点的状态。 -5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version). +5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version)。 具体的流程图如下： -
    +![replica-master.png](page://images/architecture/replica-master.png) 选择Master的具体规则如下： @@ -105,7 +105,7 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性 如果vnode A是master, vnode B是slave, vnode A能接受客户端的写请求，而vnode B不能。当vnode A收到写的请求后，遵循下面的流程： -
    +![replica-forward.png](page://images/architecture/replica-forward.png) 1. 应用对写请求做基本的合法性检查，通过，则给改请求包打上一个版本号(version, 单调递增） 2. 应用将打上版本号的写请求封装一个WAL Head, 写入WAL(Write Ahead Log) @@ -124,23 +124,23 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性 如果一虚拟节点(vnode B) 处于unsynced状态，master存在（vnode A)，而且其版本号比master的低，它将立即启动数据恢复流程。在理解恢复流程时，需要澄清几个关于文件的概念和处理规则。 -1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mnode里的文件，数量是固定的，对应于acct, user, db, table等文件。 +1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mode里的文件，数量是固定的，对应于acct, user, db, table等文件。 2. 任何一个数据文件(file)有名字、大小，还有一个magic number。只有文件名、大小与magic number一致时，两个文件才判断是一样的，无需同步。Magic number可以是checksum, 也可以是简单的文件大小。怎么计算magic，换句话说，如何检测数据文件是否有效，完全由应用决定。 3. 文件名的处理有点复杂，因为每台服务器的路径可能不一致。比如node A的TDengine的数据文件存放在 /etc/taos目录下，而node B的数据存放在 /home/jhtao目录下。因此同步模块需要应用在启动一个同步实例时提供一个path，这样两台服务器的绝对路径可以不一样，但仍然可以做对比，做同步。 4. 当sync模块调用回调函数getFileInfo获得数据文件信息时，有如下的规则 - 1. index 为0，表示获取最老的文件，同时修改index返回给sync模块。如果index不为0，表示获取指定位置的文件。 - 2. 如果name为空，表示sync想获取位于index位置的文件信息，包括magic, size。Master节点会这么调用 - 3. 如果name不为空，表示sync想获取指定文件名和index的信息，slave节点会这么调用 - 4. 如果某个index的文件不存在，magic返回0，表示文件已经是最后一个。因此整个系统里，文件的index必须是连续的一段整数。 + * index 为0，表示获取最老的文件，同时修改index返回给sync模块。如果index不为0，表示获取指定位置的文件。 + * 如果name为空，表示sync想获取位于index位置的文件信息，包括magic, size。Master节点会这么调用 + * 如果name不为空，表示sync想获取指定文件名和index的信息，slave节点会这么调用 + * 如果某个index的文件不存在，magic返回0，表示文件已经是最后一个。因此整个系统里，文件的index必须是连续的一段整数。 5. 当sync模块调用回调函数getWalInfo获得wal信息时，有如下规则 - 1. index为0，表示获得最老的WAL文件, 返回时，index更新为具体的数字 - 2. 如果返回0，表示这是最新的一个WAL文件，如果返回值是1，表示后面还有更新的WAL文件 - 3. 返回的文件名为空，那表示没有WAL文件 + * index为0，表示获得最老的WAL文件, 返回时，index更新为具体的数字 + * 如果返回0，表示这是最新的一个WAL文件，如果返回值是1，表示后面还有更新的WAL文件 + * 返回的文件名为空，那表示没有WAL文件 6. 无论是getFileInfo, 还是getWalInfo, 只要获取出错（不是文件不存在），返回-1即可，系统会报错，停止同步 整个数据恢复流程分为两大步骤，第一步，先恢复archived data(file), 然后恢复wal。具体流程如下： -
    +![replica-forward.png](page://images/architecture/replica-forward.png) 1. 通过已经建立的TCP连接，发送sync req给master节点 2. master收到sync req后，以client的身份，向vnode B主动建立一新的专用于同步的TCP连接（syncFd) @@ -212,10 +212,10 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系 相同之处： -- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程 -- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的 +- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程。 +- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的。 - 数据转发流程完全一样，Master(leader)需要等待回复确认。 -- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步 +- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步。 不同之处： @@ -226,7 +226,7 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系 ## Meta Data的数据复制 -TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析 +TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析。 TDengine里Meta Data包括以下： diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/cn/03.architecture/docs.md similarity index 92% rename from documentation20/webdocs/markdowndocs/architecture-ch.md rename to documentation20/cn/03.architecture/docs.md index c9bfa30830fe7b2f3cd1364b589326255560ad83..dc710704e50c5f6b3504a8427ae7088a15b12e77 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/cn/03.architecture/docs.md @@ -1,7 +1,9 @@ -#数据模型和整体架构 +# 数据模型和整体架构 + +## 数据模型 -## 数据模型 ### 物联网典型场景 + 在典型的物联网、车联网、运维监测场景中，往往有多种不同类型的数据采集设备，采集一个到多个不同的物理量。而同一种采集设备类型，往往又有多个具体的采集设备分布在不同的地点。大数据处理系统就是要将各种采集的数据汇总，然后进行计算和分析。对于同一类设备，其采集的数据都是很规则的。以智能电表为例，假设每个智能电表采集电流、电压、相位三个量，其采集的数据类似如下的表格：
    @@ -102,7 +104,8 @@ 每一条记录都有设备ID，时间戳，采集的物理量(如上图中的电流、电压、相位），还有与每个设备相关的静态标签（如上述表一中的位置Location和分组groupId）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。 -### 数据特征 +### 数据特征 + 除时序特征外，仔细研究发现，物联网、车联网、运维监测类数据还具有很多其他明显的特征： 1. 数据高度结构化； @@ -119,9 +122,11 @@ 充分利用上述特征，TDengine 采取了经特殊优化的存储和计算设计来处理时序数据，它将系统处理能力显著提高，同时大幅降低了系统运维的复杂度。 ### 关系型数据库模型 + 因为采集的数据一般是结构化数据，同时为降低学习门槛，TDengine采用传统的关系型数据库模型管理数据。因此用户需要先创建库，然后创建表，之后才能插入或查询数据。TDengine采用的是结构化存储，而不是NoSQL的key-value存储。 -### 一个数据采集点一张表 +### 一个数据采集点一张表 + 为充分利用其数据的时序性和其他数据特点，TDengine要求**对每个数据采集点单独建表**（比如有一千万个智能电表，就需创建一千万张表，上述表格中的d1001, d1002, d1003, d1004都需单独建表），用来存储这个采集点所采集的时序数据。这种设计有几大优点： 1. 能保证一个采集点的数据在存储介质上是以块为单位连续存储的。如果读取一个时间段的数据，它能大幅减少随机读取操作，成数量级的提升读取和查询速度。 @@ -133,6 +138,7 @@ TDengine 建议用数据采集点的名字(如上表中的D1001)来做表名。每个数据采集点可能同时采集多个物理量(如上表中的curent, voltage, phase)，每个物理量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 timestamp。对采集的数据，TDengine将自动按照时间戳建立索引，但对采集的物理量不建任何索引。数据用列式存储方式保存。 ### 超级表：同一类型数据采集点的集合 + 由于一个数据采集点一张表，导致表的数量巨增，难以管理，而且应用经常需要做采集点之间的聚合操作，聚合的操作也变得复杂起来。为解决这个问题，TDengine引入超级表(Super Table，简称为STable)的概念。 超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的结合），除需要定义采集量的表结构之外，还需要定义其标签的schema，标签的数据类型可以是整数、浮点数、字符串，标签可以有多个，可以事后增加、删除或修改。 如果整个系统有N个不同类型的数据采集点，就需要建立N个超级表。 @@ -141,16 +147,20 @@ TDengine 建议用数据采集点的名字(如上表中的D1001)来做表名。 当对多个具有相同数据类型的数据采集点进行聚合操作时，TDengine将先把满足标签过滤条件的表从超级表的中查找出来，然后再扫描这些表的时序数据，进行聚合操作，这样能将需要扫描的数据集大幅减少，从而大幅提高聚合计算的性能。 -## 集群与基本逻辑单元 +## 集群与基本逻辑单元 + TDengine 的设计是基于单个硬件、软件系统不可靠，基于任何单台计算机都无法提供足够计算能力和存储能力处理海量数据的假设进行设计的。因此 TDengine 从研发的第一天起，就按照分布式高可靠架构进行设计，是支持水平扩展的，这样任何单台或多台服务器发生硬件故障或软件错误都不影响系统的可用性和可靠性。同时，通过节点虚拟化并辅以自动化负载均衡技术，TDengine 能最高效率地利用异构集群中的计算和存储资源降低硬件投资。 ### 主要逻辑单元 + TDengine 分布式架构的逻辑结构图如下： -
    + +![TDengine架构示意图](page://images/architecture/structure.png)
    图 1 TDengine架构示意图
    + 一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine应用驱动(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。 -**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。 +**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文[《一篇文章说清楚TDengine的FQDN》](https://www.taosdata.com/blog/2020/09/11/1824.html)。 **数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。 @@ -163,26 +173,30 @@ TDengine 分布式架构的逻辑结构图如下： **TAOSC:** taosc是TDengine给应用提供的驱动程序(driver)，负责处理应用与集群的接口交互，提供C/C++语言原生接口，内嵌于JDBC、C#、Python、Go、Node.js语言连接库里。应用都是通过taosc而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据；将插入、查询等请求转发到正确的数据节点；在把结果返回给应用时，还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, C/C++/C#/Python/Go/Node.js接口而言，这个模块是在应用所处的物理节点上运行。同时，为支持全分布式的RESTful接口，taosc在TDengine集群的每个dnode上都有一运行实例。 ### 节点之间的通讯 + **通讯方式：**TDengine系统的各个数据节点之间，以及应用驱动与各数据节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。 **FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取计算机的hostname作为其FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。 -**端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP连接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 +**端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP连接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。（另外还可能有 RESTful、Arbitrator 所使用的端口，那样的话就一共是 13 个。）使用时，需要确保防火墙将这些端口打开，以备使用。每个数据节点可以配置不同的serverPort。 **集群对外连接:** TDengine集群可以容纳单个、多个甚至几千个数据节点。应用只需要向集群中任何一个数据节点发起连接即可，连接需要提供的网络参数是一数据节点的End Point(FQDN加配置的端口号）。通过命令行CLI启动应用taos时，可以通过选项-h来指定数据节点的FQDN, -P来指定其配置的端口号，如果端口不配置，将采用TDengine的系统配置参数serverPort。 -**集群内部通讯**: 各个数据节点之间通过TCP/UDP进行连接。一个数据节点启动时，将获取mnode所在的dnode的EP信息，然后与系统中的mnode建立起连接，交换信息。获取mnode的EP信息有三步，1：检查mnodeEpList文件是否存在，如果不存在或不能正常打开获得mnode EP信息，进入第二步；2：检查系统配置文件taos.cfg, 获取mnode EP配置参数first, second，如果不存在或者taos.cfg里没有这两个配置参数，或无效，进入第三步；3：将自己的EP设为mnode EP, 并独立运行起来。获取mnode EP列表后，数据节点发起连接，如果连接成功，则成功加入进工作的集群，如果不成功，则尝试mnode EP列表中的下一个。如果都尝试了，但连接都仍然失败，则休眠几秒后，再进行尝试。 +**集群内部通讯**: 各个数据节点之间通过TCP/UDP进行连接。一个数据节点启动时，将获取mnode所在的dnode的EP信息，然后与系统中的mnode建立起连接，交换信息。获取mnode的EP信息有三步，1：检查mnodeEpSet文件是否存在，如果不存在或不能正常打开获得mnode EP信息，进入第二步；2：检查系统配置文件taos.cfg, 获取节点配置参数firstEp, secondEp，（这两个参数指定的节点可以是不带mnode的普通节点，这样的话，节点被连接时会尝试重定向到mnode节点）如果不存在或者taos.cfg里没有这两个配置参数，或无效，进入第三步；3：将自己的EP设为mnode EP, 并独立运行起来。获取mnode EP列表后，数据节点发起连接，如果连接成功，则成功加入进工作的集群，如果不成功，则尝试mnode EP列表中的下一个。如果都尝试了，但连接都仍然失败，则休眠几秒后，再进行尝试。 **MNODE的选择:** TDengine逻辑上有管理节点，但没有单独的执行代码，服务器侧只有一套执行代码taosd。那么哪个数据节点会是管理节点呢？这是系统自动决定的，无需任何人工干预。原则如下：一个数据节点启动时，会检查自己的End Point, 并与获取的mnode EP List进行比对，如果在其中，该数据节点认为自己应该启动mnode模块，成为mnode。如果自己的EP不在mnode EP List里，则不启动mnode模块。在系统的运行过程中，由于负载均衡、宕机等原因，mnode有可能迁移至新的dnode，但一切都是透明的，无需人工干预，配置参数的修改，是mnode自己根据资源做出的决定。 -**新数据节点的加入**：系统有了一个数据节点后，就已经成为一个工作的系统。添加新的节点进集群时，有两个步骤，第一步：使用TDengine CLI连接到现有工作的数据节点，然后用命令”create dnode"将新的数据节点的End Point添加进去; 第二步：在新的数据节点的系统配置参数文件taos.cfg里，将first, second参数设置为现有集群中任意两个数据节点的EP即可。具体添加的详细步骤请见详细的用户手册。这样就把集群一步一步的建立起来。 +**新数据节点的加入**：系统有了一个数据节点后，就已经成为一个工作的系统。添加新的节点进集群时，有两个步骤，第一步：使用TDengine CLI连接到现有工作的数据节点，然后用命令”create dnode"将新的数据节点的End Point添加进去; 第二步：在新的数据节点的系统配置参数文件taos.cfg里，将firstEp, secondEp参数设置为现有集群中任意两个数据节点的EP即可。具体添加的详细步骤请见详细的用户手册。这样就把集群一步一步的建立起来。 **重定向**：无论是dnode还是taosc，最先都是要发起与mnode的连接，但mnode是系统自动创建并维护的，因此对于用户来说，并不知道哪个dnode在运行mnode。TDengine只要求向系统中任何一个工作的dnode发起连接即可。因为任何一个正在运行的dnode，都维护有目前运行的mnode EP List。当收到一个来自新启动的dnode或taosc的连接请求，如果自己不是mnode，则将mnode EP List回复给对方，taosc或新启动的dnode收到这个list, 就重新尝试建立连接。当mnode EP List发生改变，通过节点之间的消息交互，各个数据节点就很快获取最新列表，并通知taosc。 ### 一个典型的消息流程 + 为解释vnode, mnode, taosc和应用之间的关系以及各自扮演的角色，下面对写入数据这个典型操作的流程进行剖析。 -
    + +![TDengine典型的操作流程](page://images/architecture/message.png)
    图 2 TDengine典型的操作流程
    + 1. 应用通过JDBC、ODBC或其他API接口发起插入数据的请求。 2. taosc会检查缓存，看是否保存有该表的meta data。如果有，直接到第4步。如果没有，taosc将向mnode发出get meta-data请求。 3. mnode将该表的meta-data返回给taosc。Meta-data包含有该表的schema, 而且还有该表所属的vgroup信息（vnode ID以及所在的dnode的End Point，如果副本数为N，就有N组End Point)。如果taosc迟迟得不到mnode回应，而且存在多个mnode, taosc将向下一个mnode发出请求。 @@ -198,14 +212,15 @@ TDengine 分布式架构的逻辑结构图如下： 通过taosc缓存机制，只有在第一次对一张表操作时，才需要访问mnode,因此mnode不会成为系统瓶颈。但因为schema有可能变化，而且vgroup有可能发生改变（比如负载均衡发生），因此taosc会定时和mnode交互，自动更新缓存。 -## 存储模型与数据分区、分片 +## 存储模型与数据分区、分片 ### 存储模型 + TDengine存储的数据包括采集的时序数据以及库、表相关的元数据、标签数据等，这些数据具体分为三部分： -- 时序数据：存放于vnode里，由data、head和last三个文件组成，数据量大，查询量取决于应用场景。容许乱序写入，但暂时不支持删除和更新操作。通过采用一个采集点一张表的模型，一个时间段的数据是连续存储，对单张表的写入是简单的追加操作，一次读，可以读到多条记录，这样保证对单个采集点的插入和查询操作，性能达到最优。 +- 时序数据：存放于vnode里，由data、head和last三个文件组成，数据量大，查询量取决于应用场景。容许乱序写入，但暂时不支持删除操作，并且仅在update参数设置为1时允许更新操作。通过采用一个采集点一张表的模型，一个时间段的数据是连续存储，对单张表的写入是简单的追加操作，一次读，可以读到多条记录，这样保证对单个采集点的插入和查询操作，性能达到最优。 - 标签数据：存放于vnode里的meta文件，支持增删改查四个标准操作。数据量不大，有N张表，就有N条记录，因此可以全内存存储。如果标签过滤操作很多，查询将十分频繁，因此TDengine支持多核多线程并发查询。只要计算资源足够，即使有数千万张表，过滤结果能毫秒级返回。 -- 其他元数据：存放于mnode里，包含系统节点、用户、DB、Table Schema等等，支持增删改查四个标准操作。这部分数据的量不大，可以全内存保存，而且由于客户端有缓存，查询量也不大。因此目前的设计虽是集中式存储管理，但不会构成性能瓶颈。 +- 元数据：存放于mnode里，包含系统节点、用户、DB、Table Schema等信息，支持增删改查四个标准操作。这部分数据的量不大，可以全内存保存，而且由于客户端有缓存，查询量也不大。因此目前的设计虽是集中式存储管理，但不会构成性能瓶颈。 与典型的NoSQL存储模型相比，TDengine将标签数据与时序数据完全分离存储，它具有两大优势： @@ -213,6 +228,7 @@ TDengine存储的数据包括采集的时序数据以及库、表相关的元数 - 能够实现极为高效的多表之间的聚合查询：做多表之间聚合查询时，先把符合标签过滤条件的表查找出来，然后再查找这些表相应的数据块，这样大幅减少要扫描的数据集，从而大幅提高查询效率。而且标签数据采用全内存的结构进行管理和维护，千万级别规模的标签数据查询可以在毫秒级别返回。 ### 数据分片 + 对于海量的数据管理，为实现水平扩展，一般都需要采取分片(Sharding)分区(Partitioning)策略。TDengine是通过vnode来实现数据分片的，通过一个时间段一个数据文件来实现时序数据分区的。 vnode(虚拟数据节点)负责为采集的时序数据提供写入、查询和计算功能。为便于负载均衡、数据恢复、支持异构环境，TDengine将一个数据节点根据其计算和存储资源切分为多个vnode。这些vnode的管理是TDengine自动完成的，对应用完全透明。 @@ -224,11 +240,13 @@ vnode(虚拟数据节点)负责为采集的时序数据提供写入、查询和 每张表的meda data（包含schema, 标签等）也存放于vnode里，而不是集中存放于mnode，实际上这是对Meta数据的分片，这样便于高效并行的进行标签过滤操作。 ### 数据分区 + TDengine除vnode分片之外，还对时序数据按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由DB的配置参数days决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数keep)，将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。 总的来说，**TDengine是通过vnode以及时间两个维度，对大数据进行切分**，便于并行高效的管理，实现水平扩展。 ### 负载均衡 + 每个dnode都定时向 mnode(虚拟管理节点)报告其状态（包括硬盘空间、内存大小、CPU、网络、虚拟节点个数等），因此mnode了解整个集群的状态。基于整体状态，当mnode发现某个dnode负载过重，它会将dnode上的一个或多个vnode挪到其他dnode。在挪动过程中，对外服务继续进行，数据插入、查询和计算操作都不受影响。 如果mnode一段时间没有收到dnode的状态报告，mnode会认为这个dnode已经离线。如果离线时间超过一定时长（时长由配置参数offlineThreshold决定），该dnode将被mnode强制剔除出集群。该dnode上的vnodes如果副本数大于一，系统将自动在其他dnode上创建新的副本，以保证数据的副本数。如果该dnode上还有mnode, 而且mnode的副本数大于一，系统也将自动在其他dnode上创建新的mnode, 以保证mnode的副本数。 @@ -238,25 +256,31 @@ TDengine除vnode分片之外，还对时序数据按照时间段进行分区。 负载均衡过程无需任何人工干预，应用也无需重启，将自动连接新的节点，完全透明。 **提示：负载均衡由参数balance控制，决定开启/关闭自动负载均衡。** -## 数据写入与复制流程 +## 数据写入与复制流程 + 如果一个数据库有N个副本，那一个虚拟节点组就有N个虚拟节点，但是只有一个是Master，其他都是slave。当应用将新的记录写入系统时，只有Master vnode能接受写的请求。如果slave vnode收到写的请求，系统将通知taosc需要重新定向。 + ### Master vnode写入流程 + Master Vnode遵循下面的写入流程： -
    +![TDengine Master写入流程](page://images/architecture/write_master.png)
    图 3 TDengine Master写入流程
    + 1. Master vnode收到应用的数据插入请求，验证OK，进入下一步； 2. 如果系统配置参数walLevel大于0，vnode将把该请求的原始数据包写入数据库日志文件WAL。如果walLevel设置为2，而且fsync设置为0，TDengine还将WAL数据立即落盘，以保证即使宕机，也能从数据库日志文件中恢复数据，避免数据的丢失； 3. 如果有多个副本，vnode将把数据包转发给同一虚拟节点组内slave vnodes, 该转发包带有数据的版本号(version)； -4. 写入内存，并加记录加入到skip list； +4. 写入内存，并将记录加入到skip list； 5. Master vnode返回确认信息给应用，表示写入成功。 6. 如果第2，3，4步中任何一步失败，将直接返回错误给应用。 ### Slave vnode写入流程 + 对于slave vnode, 写入流程是： -
    +![TDengine Slave写入流程](page://images/architecture/write_slave.png)
    图 4 TDengine Slave写入流程
    + 1. Slave vnode收到Master vnode转发了的数据插入请求。 2. 如果系统配置参数walLevel大于0，vnode将把该请求的原始数据包写入数据库日志文件WAL。如果walLevel设置为2，而且fsync设置为0，TDengine还将WAL数据立即落盘，以保证即使宕机，也能从数据库日志文件中恢复数据，避免数据的丢失； 3. 写入内存，更新内存中的skip list。 @@ -264,6 +288,7 @@ Master Vnode遵循下面的写入流程： 与Master vnode相比，slave vnode不存在转发环节，也不存在回复确认环节，少了两步。但写内存与WAL是完全一样的。 ### 异地容灾、IDC迁移 + 从上述Master和Slave流程可以看出，TDengine采用的是异步复制的方式进行数据同步。这种方式能够大幅提高写入性能，网络延时对写入速度不会有大的影响。通过配置每个物理节点的IDC和机架号，可以保证对于一个虚拟节点组，虚拟节点由来自不同IDC、不同机架的物理节点组成，从而实现异地容灾。因此TDengine原生支持异地容灾，无需再使用其他工具。 另外一方面，TDengine支持动态修改副本数，一旦副本数增加，新加入的虚拟节点将立即进入数据同步流程，同步结束后，新加入的虚拟节点即可提供服务。而在同步过程中，master以及其他已经同步的虚拟节点都可以对外提供服务。利用这一特性，TDengine可以实现无服务中断的IDC机房迁移。只需要将新IDC的物理节点加入现有集群，等数据同步完成后，再将老的IDC的物理节点从集群中剔除即可。 @@ -280,6 +305,7 @@ Master Vnode遵循下面的写入流程： **提示：该功能暂未提供** ### 主从选择 + Vnode会保持一个数据版本号(Version)，对内存数据进行持久化存储时，对该版本号也进行持久化存储。每个数据更新操作，无论是采集的时序数据还是元数据，这个版本号将增一。 一个vnode启动时，角色(master、slave) 是不定的，数据是处于未同步状态，它需要与虚拟节点组内其他节点建立TCP连接，并互相交换status，其中包括version和自己的角色。通过status的交换，系统进入选主流程，规则如下： @@ -289,17 +315,20 @@ Vnode会保持一个数据版本号(Version)，对内存数据进行持久化存 3. 在线的虚拟节点数过半，而且有虚拟节点是slave的话，该虚拟节点自动成为master 4. 对于2和3，如果多个虚拟节点满足成为master的要求，那么虚拟节点组的节点列表里，最前面的选为master -更多的关于数据复制的流程，请见TDengine 2.0数据复制模块设计。 +更多的关于数据复制的流程，请见[TDengine 2.0数据复制模块设计](https://www.taosdata.com/cn/documentation/architecture/replica/)。 ### 同步复制 + 对于数据一致性要求更高的场景，异步数据复制无法满足要求，因为有极小的概率丢失数据，因此TDengine提供同步复制的机制供用户选择。在创建数据库时，除指定副本数replica之外，用户还需要指定新的参数quorum。如果quorum大于一，它表示每次Master转发给副本时，需要等待quorum-1个回复确认，才能通知应用，数据在slave已经写入成功。如果在一定的时间内，得不到quorum-1个回复确认，master vnode将返回错误给应用。 采用同步复制，系统的性能会有所下降，而且latency会增加。因为元数据要强一致，mnode之间的数据同步缺省就是采用的同步复制。 注：vnode之间的同步复制仅仅企业版支持 -## 缓存与持久化 +## 缓存与持久化 + ### 缓存 + TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心的是刚产生的数据，即当前状态。TDengine充分利用这一特性，将最近到达的（当前状态）数据保存在缓存中。 TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，**可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署Redis或其他额外的缓存系统**，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。 @@ -307,6 +336,7 @@ TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接 每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当三分之一以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有三分之一内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。 ### 持久化存储 + TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时，为了不阻塞后续数据的写入，TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件，在落盘成功后则会删除老的数据库日志文件，避免日志文件无限制的增长。 为充分利用时序数据特点，TDengine将一个vnode保存在持久化存储的数据切分成多个文件，每个文件只保存固定天数的数据，这个天数由系统配置参数days决定。切分成多个文件后，给定查询的起止日期，无需任何索引，就可以立即定位需要打开哪些数据文件，大大加快读取速度。 @@ -322,6 +352,7 @@ TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久 数据写入磁盘时，根据系统配置参数comp决定是否压缩数据。TDengine提供了三种压缩选项：无压缩、一阶段压缩和两阶段压缩，分别对应comp值为0、1和2的情况。一阶段压缩根据数据的类型进行了相应的压缩，压缩算法包括delta-delta编码、simple 8B方法、zig-zag编码、LZ4等算法。二阶段压缩在一阶段压缩的基础上又用通用压缩算法进行了压缩，压缩率更高。 ### 多级存储 + 在默认配置下，TDengine会将所有数据保存在/var/lib/taos目录下，而且每个vnode的数据文件保存在该目录下的不同目录。为扩大存储空间，尽量减少文件读取的瓶颈，提高数据吞吐率 TDengine可通过配置系统参数dataDir让多个挂载的硬盘被系统同时使用。除此之外，TDengine也提供了数据分级存储的功能，即根据数据文件的新老程度存储在不同的存储介质上。比如最新的数据存储在SSD上，超过一周的数据存储在本地硬盘上，超过4周的数据存储在网络存储设备上，这样来降低存储成本，而又保证高效的访问数据。数据在不同存储介质上的移动是由系统自动完成的，对应用是完全透明的。数据的分级存储也是通过系统参数dataDir来配置。 dataDir的配置格式如下： @@ -344,19 +375,20 @@ dataDir /mnt/disk6/taos 2 挂载的盘也可以是非本地的网络盘，只要系统能访问即可。 注：多级存储功能仅企业版支持 -**提示：该功能暂未提供** -## 数据查询 +## 数据查询 + TDengine提供了多种多样针对表和超级表的查询处理功能，除了常规的聚合查询之外，还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。 ### 单表查询 + SQL语句的解析和校验工作在客户端完成。解析SQL语句并生成抽象语法树(Abstract Syntax Tree, AST)，然后对其进行校验和检查。以及向管理节点(mnode)请求查询中指定表的元数据信息(table metadata)。 根据元数据信息中的End Point信息，将查询请求序列化后发送到该表所在的数据节点（dnode）。dnode接收到查询请求后，识别出该查询请求指向的虚拟节点（vnode），将消息转发到vnode的查询执行队列。vnode的查询执行线程建立基础的查询执行环境，并立即返回该查询请求，同时开始执行该查询。 客户端在获取查询结果的时候，dnode的查询执行队列中的工作线程会等待vnode执行线程执行完成，才能将查询结果返回到请求的客户端。 -### 按时间轴聚合、降采样、插值 +### 按时间轴聚合、降采样、插值 时序数据有别于普通数据的显著特征是每条记录均具有时间戳，因此针对具有时间戳数据在时间轴上进行聚合是不同于普通数据库的重要功能。从这点上来看，与流计算引擎的窗口查询有相似的地方。 @@ -372,21 +404,24 @@ select count(*) from d1001 interval(1h); select count(*) from d1001 interval(1h) fill(prev); ``` -针对d1001设备采集数据统计每小时记录数，如果某一个小时不存在数据，这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。 +针对d1001设备采集数据统计每小时记录数，如果某一个小时不存在数据，则返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。 ### 多表聚合查询 + TDengine对每个数据采集点单独建表，但在实际应用中经常需要对不同的采集点数据进行聚合。为高效的进行聚合操作，TDengine引入超级表（STable）的概念。超级表用来代表一特定类型的数据采集点，它是包含多张表的表集合，集合里每张表的模式（schema）完全一致，但每张表都带有自己的静态标签，标签可以多个，可以随时增加、删除和修改。 应用可通过指定标签的过滤条件，对一个STable下的全部或部分表进行聚合或统计操作，这样大大简化应用的开发。其具体流程如下图所示： -
    +![多表聚合查询原理图](page://images/architecture/multi_tables.png)
    图 5 多表聚合查询原理图
    -1：应用将一个查询条件发往系统； -2: taosc将超级表的名字发往 Meta Node（管理节点)； -3：管理节点将超级表所拥有的 vnode 列表发回 taosc； -4：taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点； -5：每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合，然后扫描存储的时序数据，完成相应的聚合计算，将结果返回给taosc； -6：taosc将多个数据节点返回的结果做最后的聚合，将其返回给应用。 + +1. 应用将一个查询条件发往系统； +2. taosc将超级表的名字发往 Meta Node（管理节点)； +3. 管理节点将超级表所拥有的 vnode 列表发回 taosc； +4. taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点； +5. 每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合，然后扫描存储的时序数据，完成相应的聚合计算，将结果返回给taosc； +6. taosc将多个数据节点返回的结果做最后的聚合，将其返回给应用。 由于TDengine在vnode内将标签数据与时序数据分离存储，通过在内存里过滤标签数据，先找到需要参与聚合操作的表的集合，将需要扫描的数据集大幅减少，大幅提升聚合计算速度。同时，由于数据分布在多个vnode/dnode，聚合计算操作在多个vnode里并发进行，又进一步提升了聚合的速度。 对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样，细节请看 TAOS SQL。 ### 预计算 + 为有效提升查询处理的性能，针对物联网数据的不可更改的特点，在数据块头部记录该数据块中存储数据的统计信息：包括最大值、最小值、和。我们称之为预计算单元。如果查询处理涉及整个数据块的全部数据，直接使用预计算结果，完全不需要读取数据块的内容。由于预计算数据量远小于磁盘上存储的数据块数据的大小，对于磁盘IO为瓶颈的查询处理，使用预计算结果可以极大地减小读取IO压力，加速查询处理的流程。预计算机制与Postgre SQL的索引BRIN（block range index）有异曲同工之妙。 diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/cn/04.model/docs.md similarity index 71% rename from documentation20/webdocs/markdowndocs/Model-ch.md rename to documentation20/cn/04.model/docs.md index c0b64bde6e5b4ee8842931751cc64b726c991321..6f85381588c6cd131f2204bba13e64e7058f93f2 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/cn/04.model/docs.md @@ -4,18 +4,20 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个具体的应用场景，需要考虑库的设计，超级表和普通表的设计。本节不讨论细致的语法规则，只介绍概念。 -## 创建库 +关于数据建模请参考[视频教程](https://www.taosdata.com/blog/2020/11/11/1945.html)。 -不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： +## 创建库 -```cmd -CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4; +不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： + +```mysql +CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4 UPDATE 1; ``` -上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4。详细的语法及参数请见TAOS SQL +上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4，允许更新数据。详细的语法及参数请见 [TAOS SQL 的数据管理](https://www.taosdata.com/cn/documentation/taos-sql#management) 章节。 创建库之后，需要使用SQL命令USE将当前库切换过来，例如： -```cmd +```mysql USE power; ``` @@ -26,23 +28,31 @@ USE power; - 任何一张表或超级表是属于一个库的，在创建表之前，必须先创建库。 - 处于两个不同库的表是不能进行JOIN操作的。 -## 创建超级表 +## 创建超级表 + 一个物联网系统，往往存在多种类型的设备，比如对于电网，存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合，使用TDengine, 需要对每个类型的数据采集点创建一超级表。以表一中的智能电表为例，可以使用如下的SQL命令创建超级表： -```cmd -CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); + +```mysql +CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); ``` -与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义。第一列必须为时间戳（示例中为ts)，其他列为采集的物理量（示例中为current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 + +**注意：**这一指令中的 STABLE 关键字，在 2.0.15 之前的版本中需写作 TABLE 。 + +与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义。第一列必须为时间戳（示例中为ts)，其他列为采集的物理量（示例中为current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 [TAOS SQL 的超级表管理](https://www.taosdata.com/cn/documentation/taos-sql#super-table) 章节。 每一种类型的数据采集点需要建立一个超级表，因此一个物联网系统，往往会有多个超级表。对于电网，我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中，一个设备就可能有多个数据采集点（比如一台风力发电的风机，有的采集点采集电流、电压等电参数，有的采集点采集温度、湿度、风向等环境参数），这个时候，对这一类型的设备，需要建立多张超级表。一张超级表里包含的采集物理量必须是同时采集的（时间戳是一致的）。 一张超级表最多容许1024列，如果一个采集点采集的物理量个数超过1024，需要建多张超级表来处理。一个系统可以有多个DB，一个DB里可以有一到多个超级表。 -## 创建表 +## 创建表 + TDengine对每个数据采集点需要独立建表。与标准的关系型数据一样，一张表有表名，Schema，但除此之外，还可以带有一到多个标签。创建时，需要使用超级表做模板，同时指定标签的具体值。以表一中的智能电表为例，可以使用如下的SQL命令建表： -```cmd + +```mysql CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2); ``` -其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 TAOS SQL。 + +其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 [TAOS SQL 的表管理](https://www.taosdata.com/cn/documentation/taos-sql#table) 章节。 **注意：**目前 TDengine 没有从技术层面限制使用一个 database （dbA）的超级表作为模板建立另一个 database （dbB）的子表，后续会禁止这种用法，不建议使用这种方法建表。 @@ -50,12 +60,17 @@ TDengine建议将数据采集点的全局唯一ID作为表名(比如设备序列 **自动建表**：在某些特殊场景中，用户在写数据时并不确定某个数据采集点的表是否存在，此时可在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。比如： -```cmd +```mysql INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32); ``` -上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。 + +上述SQL语句将记录 (now, 10.2, 219, 0.32) 插入表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。 + +关于自动建表的详细语法请参见 [插入记录时自动建表](https://www.taosdata.com/cn/documentation/taos-sql#auto_create_table) 章节。 ## 多列模型 vs 单列模型 + TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。 TDengine建议尽可能采用多列模型，因为插入效率以及存储效率更高。但对于有些场景，一个采集点的采集量的种类经常变化，这个时候，如果采用多列模型，就需要频繁修改超级表的结构定义，让应用变的复杂，这个时候，采用单列模型会显得简单。 + diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/cn/05.insert/docs.md similarity index 88% rename from documentation20/webdocs/markdowndocs/insert-ch.md rename to documentation20/cn/05.insert/docs.md index 77ba596d4ec6de76a39b878e57ea255b081fea45..ce2d65e7d2259c6dac9efc67a61f7c009dd96984 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/cn/05.insert/docs.md @@ -2,7 +2,7 @@ TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。 -## SQL写入 +## SQL写入 应用通过C/C++, JDBC, GO, 或Python Connector 执行SQL insert语句来插入数据，用户还可以通过TAOS Shell，手动输入SQL insert语句插入数据。比如下面这条insert 就将一条记录写入到表d1001中： ```mysql @@ -18,26 +18,28 @@ TDengine也支持一次向多个表写入数据，比如下面这条命令就向 INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) d1002 VALUES (1538548696800, 12.3, 221, 0.31); ``` -详细的SQL INSERT语法规则请见TAOS SQL +详细的SQL INSERT语法规则请见 [TAOS SQL 的数据写入](https://www.taosdata.com/cn/documentation/taos-sql#insert) 章节。 **Tips:** - 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置，最大可配置为1M）。 - TDengine支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程切频繁切换，带来额外开销。 -- 对同一张表，如果新插入记录的时间戳已经存在，新记录将被直接抛弃，也就是说，在一张表里，时间戳必须是唯一的。如果应用自动生成记录，很有可能生成的时间戳是一样的，这样，成功插入的记录条数会小于应用插入的记录条数。 +- 对同一张表，如果新插入记录的时间戳已经存在，默认（没有使用 UPDATE 1 创建数据库）新记录将被直接抛弃，也就是说，在一张表里，时间戳必须是唯一的。如果应用自动生成记录，很有可能生成的时间戳是一样的，这样，成功插入的记录条数会小于应用插入的记录条数。如果在创建数据库时使用 UPDATE 1 选项，插入相同时间戳的新记录将覆盖原有记录。 - 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天，那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2，那么无法写入比当前时间还晚2天的数据。 -## Prometheus直接写入 +## Prometheus直接写入 + [Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 ### 从源代码编译blm_prometheus + 用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件： - Linux操作系统的服务器 - 安装好Golang, 1.10版本以上 - 对应的TDengine版本。因为用到了TDengine的客户端动态链接库，因此需要安装好和服务端相同版本的TDengine程序；比如服务端版本是TDengine 2.0.0, 则在bailongma所在的linux服务器（可以与TDengine在同一台服务器，或者不同服务器） Bailongma项目中有一个文件夹blm_prometheus，存放了prometheus的写入API程序。编译过程如下： -``` +```bash cd blm_prometheus go build ``` @@ -45,10 +47,12 @@ go build 一切正常的情况下，就会在对应的目录下生成一个blm_prometheus的可执行程序。 ### 安装Prometheus + 通过Prometheus的官网下载安装。[下载地址](https://prometheus.io/download/) ### 配置Prometheus -参考Prometheus的[配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/),在Prometheus的配置文件中的部分，增加以下配置 + +参考Prometheus的[配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/)，在Prometheus的配置文件中的部分，增加以下配置 - url: bailongma API服务提供的URL, 参考下面的blm_prometheus启动示例章节 @@ -60,7 +64,7 @@ blm_prometheus程序有以下选项，在启动blm_prometheus程序时可以通 --tdengine-name 如果TDengine安装在一台具备域名的服务器上，也可以通过配置TDengine的域名来访问TDengine。在K8S环境下，可以配置成TDengine所运行的service name ---batch-size +--batch-size blm_prometheus会将收到的prometheus的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。 --dbname @@ -79,7 +83,7 @@ blm_prometheus对prometheus提供服务的端口号。 ### 启动示例 通过以下命令启动一个blm_prometheus的API服务 -``` +```bash ./blm_prometheus -port 8088 ``` 假设blm_prometheus所在服务器的IP地址为"10.1.2.3"，则在prometheus的配置文件中部分增加url为 @@ -107,15 +111,17 @@ prometheus产生的数据格式如下： } ``` 其中，apiserver_request_latencies_bucket为prometheus采集的时序数据的名称，后面{}中的为该时序数据的标签。blm_prometheus会以时序数据的名称在TDengine中自动创建一个超级表，并将{}中的标签转换成TDengine的tag值，Timestamp作为时间戳，value作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。 -``` +```mysql use prometheus; select * from apiserver_request_latencies_bucket; ``` -## Telegraf直接写入 +## Telegraf直接写入 + [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)是一流行的IT运维数据采集开源工具，TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 ### 从源代码编译blm_telegraf + 用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件： - Linux操作系统的服务器 @@ -124,7 +130,7 @@ select * from apiserver_request_latencies_bucket; Bailongma项目中有一个文件夹blm_telegraf，存放了Telegraf的写入API程序。编译过程如下： -``` +```bash cd blm_telegraf go build ``` @@ -132,12 +138,14 @@ go build 一切正常的情况下，就会在对应的目录下生成一个blm_telegraf的可执行程序。 ### 安装Telegraf -目前TDengine支持Telegraf 1.7.4以上的版本。用户可以根据当前的操作系统，到Telegraf官网下载安装包，并执行安装。下载地址如下：https://portal.influxdata.com/downloads + +目前TDengine支持Telegraf 1.7.4以上的版本。用户可以根据当前的操作系统，到Telegraf官网下载安装包，并执行安装。下载地址如下：https://portal.influxdata.com/downloads ### 配置Telegraf + 修改Telegraf配置文件/etc/telegraf/telegraf.conf中与TDengine有关的配置项。 -在output plugins部分，增加[[outputs.http]]配置项： +在output plugins部分，增加[[outputs.http]]配置项： - url： bailongma API服务提供的URL, 参考下面的启动示例章节 - data_format: "json" @@ -154,10 +162,10 @@ go build blm_telegraf程序有以下选项，在启动blm_telegraf程序时可以通过设定这些选项来设定blm_telegraf的配置。 ```sh ---host +--host TDengine服务端的IP地址，缺省值为空 ---batch-size +--batch-size blm_telegraf会将收到的telegraf的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。 --dbname @@ -174,8 +182,9 @@ blm_telegraf对telegraf提供服务的端口号。 ``` ### 启动示例 + 通过以下命令启动一个blm_telegraf的API服务 -``` +```bash ./blm_telegraf -host 127.0.0.1 -port 8089 ``` @@ -186,6 +195,7 @@ url = "http://10.1.2.3:8089/telegraf" ``` ### 查询telegraf写入数据 + telegraf产生的数据格式如下： ```json { @@ -213,20 +223,17 @@ telegraf产生的数据格式如下： 其中，name字段为telegraf采集的时序数据的名称，tags字段为该时序数据的标签。blm_telegraf会以时序数据的名称在TDengine中自动创建一个超级表，并将tags字段中的标签转换成TDengine的tag值，Timestamp作为时间戳，fields字段中的值作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。 -``` +```mysql use telegraf; select * from cpu; ``` - - MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。 -## EMQ Broker 直接写入 +## EMQ Broker 直接写入 -[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 +[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDengine 驱动实现直接保存。详细使用方法请参考[EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 -## HiveMQ Broker 直接写入 +## HiveMQ Broker 直接写入 [HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。 - diff --git a/documentation20/webdocs/markdowndocs/Queries-ch.md b/documentation20/cn/06.queries/docs.md similarity index 89% rename from documentation20/webdocs/markdowndocs/Queries-ch.md rename to documentation20/cn/06.queries/docs.md index 960bb39e6323fe590905de62d2a1021adb19d176..a161778a72728ca05a75538c8b04ca0277e01bb2 100644 --- a/documentation20/webdocs/markdowndocs/Queries-ch.md +++ b/documentation20/cn/06.queries/docs.md @@ -1,15 +1,12 @@ - - - # 高效查询数据 -## 主要查询功能 +## 主要查询功能 TDengine 采用 SQL 作为查询语言。应用程序可以通过 C/C++, Java, Go, Python 连接器发送 SQL 语句，用户可以通过 TDengine 提供的命令行（Command Line Interface, CLI）工具 TAOS Shell 手动执行 SQL 即席查询（Ad-Hoc Query）。TDengine 支持如下查询功能： - 单列、多列数据查询 -- 标签和数值的多种过滤条件：\>, \<, =, \<>, like 等 +- 标签和数值的多种过滤条件：>, <, =, <>, like 等 - 聚合结果的分组（Group by）、排序（Order by）、约束输出（Limit/Offset） - 数值列及聚合结果的四则运算 - 时间戳对齐的连接查询（Join Query: 隐式连接）操作 @@ -26,9 +23,10 @@ Query OK, 2 row(s) in set (0.001100s) ``` 为满足物联网场景的需求，TDengine支持几个特殊的函数，比如twa(时间加权平均)，spread (最大值与最小值的差)，last_row(最后一条记录)等，更多与物联网场景相关的函数将添加进来。TDengine还支持连续查询。 -具体的查询语法请看TAOS SQL 。 +具体的查询语法请看 [TAOS SQL 的数据查询](https://www.taosdata.com/cn/documentation/taos-sql#select) 章节。 + +## 多表聚合查询 -## 多表聚合查询 物联网场景中，往往同一个类型的数据采集点有多个。TDengine采用超级表(STable)的概念来描述某一个类型的数据采集点，一张普通的表来描述一个具体的数据采集点。同时TDengine使用标签来描述数据采集点的静态属性，一个具体的数据采集点有具体的标签值。通过指定标签的过滤条件，TDengine提供了一高效的方法将超级表(某一类型的数据采集点)所属的子表进行聚合查询。对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样。 **示例1**：在TAOS Shell，查找北京所有智能电表采集的电压平均值，并按照location分组 @@ -51,9 +49,9 @@ taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - Query OK, 1 row(s) in set (0.002136s) ``` -TDengine仅容许对属于同一个超级表的表之间进行聚合查询，不同超级表之间的聚合查询不支持。在TAOS SQL 一章，查询类操作都会注明是否支持超级表。 +TDengine仅容许对属于同一个超级表的表之间进行聚合查询，不同超级表之间的聚合查询不支持。在 [TAOS SQL 的数据查询](https://www.taosdata.com/cn/documentation/taos-sql#select) 一章，查询类操作都会注明是否支持超级表。 -## 降采样查询、插值 +## 降采样查询、插值 物联网场景里，经常需要通过降采样（down sampling）将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如，将智能电表 d1001 采集的电流值每10秒钟求和 ```mysql @@ -91,5 +89,5 @@ Query OK, 5 row(s) in set (0.001521s) 物联网场景里，每个数据采集点采集数据的时间是难同步的，但很多分析算法(比如FFT)需要把采集的数据严格按照时间等间隔的对齐，在很多系统里，需要应用自己写程序来处理，但使用TDengine的降采样操作就轻松解决。如果一个时间间隔里，没有采集的数据，TDengine还提供插值计算的功能。 -语法规则细节请见TAOS SQL 。 +语法规则细节请见 [TAOS SQL 的时间维度聚合](https://www.taosdata.com/cn/documentation/taos-sql#aggregation) 章节。 diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/cn/07.advanced-features/docs.md similarity index 94% rename from documentation20/webdocs/markdowndocs/advanced features-ch.md rename to documentation20/cn/07.advanced-features/docs.md index ed02af25f2cd78372f20a2dc074bb0aa30b4e301..bdf93fdc3d74184bd7a6fd6f4eefaf3db6853c22 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/cn/07.advanced-features/docs.md @@ -1,6 +1,6 @@ # 高级功能 -## 连续查询(Continuous Query) +## 连续查询(Continuous Query) 连续查询是TDengine定期自动执行的查询，采用滑动窗口的方式进行计算，是一种简化的时间驱动的流式计算。 针对库中的表或超级表，TDengine可提供定期自动执行的连续查询， @@ -17,10 +17,8 @@ TDengine提供的连续查询与普通流计算中的时间窗口计算具有以 - 不同于流计算的实时反馈计算结果，连续查询只在时间窗口关闭以后才开始计算。 例如时间周期是1天，那么当天的结果只会在23:59:59以后才会生成。 - - 如果有历史记录写入到已经计算完成的时间区间，连续查询并不会重新进行计算， 也不会重新将结果推送给用户。对于写回TDengine的模式，也不会更新已经存在的计算结果。 - - 使用连续查询推送结果的模式，服务端并不缓存客户端计算状态，也不提供Exactly-Once的语意保证。 如果用户的应用端崩溃，再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。 如果使用写回模式，TDengine可确保数据写回的有效性和连续性。 @@ -60,7 +58,7 @@ create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s 会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句， 并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如： -```shell +```mysql taos> select * from avg_vol; ts | avg_voltage_ | =================================================== @@ -72,14 +70,13 @@ taos> select * from avg_vol; 需要注意，查询时间窗口的最小值是10毫秒，没有时间窗口范围的上限。 - 此外，TDengine还支持用户指定连续查询的起止时间。 如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始； 如果没有输入结束时间，连续查询将永久运行； 如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。 比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。 -```sql +```mysql create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s); ``` @@ -96,7 +93,7 @@ create table avg_vol as select avg(voltage) from meters where ts > now and ts <= 后续版本会提供更细粒度和便捷的连续查询管理命令。 -## 数据订阅(Publisher/Subscriber) +## 数据订阅(Publisher/Subscriber) 基于数据天然的时间序列特性，TDengine的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致， 均可视为系统中插入一条带时间戳的新记录。 @@ -119,7 +116,7 @@ taos_consume taos_unsubscribe ``` -这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation20/connector/)， +这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation/connector/)， 下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”）， 完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 @@ -198,7 +195,7 @@ select * from meters where ts > now - 1d and current > 10; 且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 `taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。 -在同步模式下，如过前后两次调用`taos_consume`的时间间隔小于此时间， +在同步模式下，如果前后两次调用`taos_consume`的时间间隔小于此时间， `taos_consume`会阻塞，直到间隔超过此时间。 异步模式下，这个时间是两次调用回调函数的最小时间间隔。 @@ -297,7 +294,7 @@ $ taos ### Java 使用数据订阅功能 -订阅功能也提供了 Java 开发接口，相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation20/connector/)。需要注意的是，目前 Java 接口没有提供异步订阅模式，但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 +订阅功能也提供了 Java 开发接口，相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation/connector/)。需要注意的是，目前 Java 接口没有提供异步订阅模式，但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 下面以一个示例程序介绍其具体使用方法。它所完成的功能与前面介绍的 C 语言示例基本相同，也是订阅数据库中所有电流超过 10A 的记录。 @@ -407,7 +404,7 @@ ts: 1597466400000 current: 12.4 voltage: 220 phase: 1 location: Beijing.Chaoyang ``` -## 缓存(Cache) +## 缓存(Cache) TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Use，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 @@ -415,7 +412,7 @@ TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接 TDengine分配固定大小的内存空间作为缓存空间，缓存空间可根据应用的需求和硬件资源配置。通过适当的设置缓存空间，TDengine可以提供极高性能的写入和查询的支持。TDengine中每个虚拟节点（virtual node）创建时分配独立的缓存池。每个虚拟节点管理自己的缓存池，不同虚拟节点间不共享缓存池。每个虚拟节点内部所属的全部表共享该虚拟节点的缓存池。 -TDengine将内存池按块划分进行管理，数据在内存块里按照列式存储。一个vnode的内存池是在vnode创建时按块分配好的，而且每个内存块按照先进先出的原则进行管理。一张表所需要的内存块是从vnode的内存池中进行分配的，块的大小由系统配置参数cache决定。每张表最大内存块的数目由配置参数tblocks决定，每张表平均的内存块的个数由配置参数ablocks决定。因此对于一个vnode, 总的内存大小为: `cache * ablocks * tables`。内存块参数cache不宜过小，一个cache block需要能存储至少几十条以上记录，才会有效率。参数ablocks最小为2，保证每张表平均至少能分配两个内存块。 +TDengine将内存池按块划分进行管理，数据在内存块里是以行（row）的形式存储。一个vnode的内存池是在vnode创建时按块分配好，而且每个内存块按照先进先出的原则进行管理。在创建内存池时，块的大小由系统配置参数cache决定；每个vnode中内存块的数目则由配置参数blocks决定。因此对于一个vnode，总的内存大小为：`cache * blocks`。一个cache block需要保证每张表能存储至少几十条以上记录，才会有效率。 你可以通过函数last_row快速获取一张表或一张超级表的最后一条记录，这样很便于在大屏显示各设备的实时状态或采集值。例如： @@ -426,7 +423,7 @@ select last_row(voltage) from meters where location='Beijing.Chaoyang'; 该SQL语句将获取所有位于北京朝阳区的电表最后记录的电压值。 -## 报警监测(Alert) +## 报警监测(Alert) 在 TDengine 的应用场景中，报警监测是一个常见需求，从概念上说，它要求程序从最近一段时间的数据中筛选出符合一定条件的数据，并基于这些数据根据定义好的公式计算出一个结果，当这个结果符合某个条件且持续一定时间后，以某种形式通知用户。 diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/cn/08.connector/01.java/docs.md similarity index 50% rename from documentation20/webdocs/markdowndocs/connector-java-ch.md rename to documentation20/cn/08.connector/01.java/docs.md index e3743f4f9de727f396e343abac6234f10265cd78..c39f6ffb4c98e97c21369b5767eff38f0fa9fe3c 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -1,54 +1,62 @@ # Java Connector -TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。 +TDengine 提供了遵循 JDBC 标准（3.0）API 规范的 `taos-jdbcdriver` 实现，可在 maven 的中央仓库 [Sonatype Repository][1] 搜索下载。 -由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 +`taos-jdbcdriver` 的实现包括 2 种形式： JDBC-JNI 和 JDBC-RESTful（taos-jdbcdriver-2.0.18 开始支持 JDBC-RESTful）。 JDBC-JNI 通过调用客户端 libtaos.so（或 taos.dll ）的本地方法实现， JDBC-RESTful 则在内部封装了 RESTful 接口实现。 -* libtaos.so - 在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 - -* taos.dll - 在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 - -> 注意：在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 - -TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点： - -* TDengine 不提供针对单条数据记录的删除和修改的操作，驱动中也没有支持相关方法。 -* 由于不支持删除和修改，所以也不支持事务操作。 -* 目前不支持表间的 union 操作。 -* 目前不支持嵌套查询(nested query)，对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet。 +![tdengine-connector](page://images/tdengine-jdbc-connector.png) +上图显示了 3 种 Java 应用使用连接器访问 TDengine 的方式： -## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 +* JDBC-JNI：Java 应用在物理节点1（pnode1）上使用 JDBC-JNI 的 API ，直接调用客户端 API（libtaos.so 或 taos.dll）将写入和查询请求发送到位于物理节点2（pnode2）上的 taosd 实例。 +* RESTful：应用将 SQL 发送给位于物理节点2（pnode2）上的 RESTful 连接器，再调用客户端 API（libtaos.so）。 +* JDBC-RESTful：Java 应用通过 JDBC-RESTful 的 API ，将 SQL 封装成一个 RESTful 请求，发送给物理节点2的 RESTful 连接器。 -| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | -| --- | --- | --- | -| 2.0.4 | 2.0.0.x 及以上 | 1.8.x | -| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | +TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征存在差异，导致 `taos-jdbcdriver` 与传统的 JDBC driver 也存在一定差异。在使用时需要注意以下几点： -## TDengine DataType 和 Java DataType - -TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： - -| TDengine DataType | Java DataType | -| --- | --- | -| TIMESTAMP | java.sql.Timestamp | -| INT | java.lang.Integer | -| BIGINT | java.lang.Long | -| FLOAT | java.lang.Float | -| DOUBLE | java.lang.Double | -| SMALLINT, TINYINT |java.lang.Short | -| BOOL | java.lang.Boolean | -| BINARY, NCHAR | java.lang.String | - -## 如何获取 TAOS-JDBCDriver +* TDengine 目前不支持针对单条数据记录的删除操作。 +* 目前不支持事务操作。 +* 目前不支持表间的 union 操作。 +* 目前不支持嵌套查询（nested query）。 +* 对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet 还没关闭的情况下执行了新的查询，taos-jdbcdriver 会自动关闭上一个 ResultSet。 + + +## JDBC-JNI和JDBC-RESTful的对比 + +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    对比项JDBC-JNIJDBC-RESTful
    支持的操作系统linux、windows全平台
    是否需要安装 client需要不需要
    server 升级后是否需要升级 client需要不需要
    写入性能JDBC-RESTful 是 JDBC-JNI 的 50%～90%
    查询性能JDBC-RESTful 与 JDBC-JNI 没有差别
    + + +## 如何获取 taos-jdbcdriver ### maven 仓库 目前 taos-jdbcdriver 已经发布到 [Sonatype Repository][1] 仓库，且各大仓库都已同步。 + * [sonatype][8] * [mvnrepository][9] * [maven.aliyun][10] @@ -59,31 +67,63 @@ maven 项目中使用如下 pom.xml 配置即可： com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.18 ``` ### 源码编译打包 -下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。 +下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package -Dmaven.test.skip=true` 即可生成相应 jar 包。 + -## 使用说明 +## JDBC的使用说明 ### 获取连接 -#### 通过JdbcUrl获取连接 +#### 指定URL获取连接 + +通过指定URL获取连接，如下所示： + +```java +Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); +String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` + +以上示例，使用 **JDBC-RESTful** 的 driver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。 + +使用 JDBC-RESTful 接口，不需要依赖本地函数库。与 JDBC-JNI 相比，仅需要： + +1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”； +2. jdbcUrl 以“jdbc:TAOS-RS://”开头； +3. 使用 6041 作为连接端口。 + +如果希望获得更好的写入和查询性能，Java 应用可以使用 **JDBC-JNI** 的driver，如下所示： -通过指定的jdbcUrl获取连接，如下所示： ```java Class.forName("com.taosdata.jdbc.TSDBDriver"); String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(jdbcUrl); ``` -以上示例，建立了到hostname为taosdemo.com，端口为6030（TDengine的默认端口），数据库名为test的连接。这个url中指定用户名（user）为root，密码（password）为taosdata。 + +以上示例，使用了 JDBC-JNI 的 driver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。 + +**注意**：使用 JDBC-JNI 的 driver，taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 + +* libtaos.so + 在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 + +* taos.dll + 在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 + +> 在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 + +JDBC-JNI 的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 TDengine 的 JDBC URL 规范格式为： -`jdbc:TAOS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]` +`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]` + url中的配置参数如下： * user：登录 TDengine 用户名，默认值 root。 * password：用户登录密码，默认值 taosdata。 @@ -92,13 +132,17 @@ url中的配置参数如下： * locale：客户端语言环境，默认值系统当前 locale。 * timezone：客户端使用的时区，默认值为系统当前时区。 -#### 使用JdbcUrl和Properties获取连接 -除了通过指定的jdbcUrl获取连接，还可以使用Properties指定建立连接时的参数，如下所示： + +#### 指定URL和Properties获取连接 + +除了通过指定的 URL 获取连接，还可以使用 Properties 指定建立连接时的参数，如下所示： ```java public Connection getConn() throws Exception{ Class.forName("com.taosdata.jdbc.TSDBDriver"); + // Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; + // String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); @@ -107,9 +151,10 @@ public Connection getConn() throws Exception{ return conn; } ``` -以上示例，建立一个到hostname为taosdemo.com，端口为6030，数据库名为test的连接。这个连接在url中指定了用户名(user)为root，密码（password）为taosdata，并在connProps中指定了使用的字符集、语言环境、时区等信息。 -properties中的配置参数如下： +以上示例，建立一个到 hostname 为 taosdemo.com，端口为 6030，数据库名为 test 的连接。注释为使用 JDBC-RESTful 时的方法。这个连接在 url 中指定了用户名(user)为 root，密码（password）为 taosdata，并在 connProps 中指定了使用的字符集、语言环境、时区等信息。 + +properties 中的配置参数如下： * TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 root。 * TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 taosdata。 * TSDBDriver.PROPERTY_KEY_CONFIG_DIR：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。 @@ -117,10 +162,15 @@ properties中的配置参数如下： * TSDBDriver.PROPERTY_KEY_LOCALE：客户端语言环境，默认值系统当前 locale。 * TSDBDriver.PROPERTY_KEY_TIME_ZONE：客户端使用的时区，默认值为系统当前时区。 + + #### 使用客户端配置文件建立连接 -当使用JDBC连接TDengine集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的firstEp、secondEp参数。 + +当使用 JDBC-JNI 连接 TDengine 集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的 firstEp、secondEp参数。 如下所示： -1. 在java中不指定hostname和port + +1. 在 Java 应用中不指定 hostname 和 port + ```java public Connection getConn() throws Exception{ Class.forName("com.taosdata.jdbc.TSDBDriver"); @@ -133,7 +183,9 @@ public Connection getConn() throws Exception{ return conn; } ``` -2. 在配置文件中指定firstEp和secondEp + +2. 在配置文件中指定 firstEp 和 secondEp + ``` # first fully qualified domain name (FQDN) for TDengine system firstEp cluster_node1:6030 @@ -142,23 +194,25 @@ firstEp cluster_node1:6030 secondEp cluster_node2:6030 # default system charset -# charset UTF-8 +# charset UTF-8 # system locale # locale en_US.UTF-8 ``` -以上示例，jdbc会使用客户端的配置文件，建立到hostname为cluster_node1，端口为6030，数据库名为test的连接。当集群中firstEp节点失效时，JDBC会尝试使用secondEp连接集群。 -TDengine中，只要保证firstEp和secondEp中一个节点有效，就可以正常建立到集群的连接。 +以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp 连接集群。 +TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。 -> 注意：这里的配置文件指的是调用JDBC Connector的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 +> 注意：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 #### 配置参数的优先级 -通过以上3种方式获取连接，如果配置参数在url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下： + +通过以上 3 种方式获取连接，如果配置参数在 url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下： 1. JDBC URL 参数，如上所述，可以在 JDBC URL 的参数中指定。 2. Properties connProps 3. 客户端配置文件 taos.cfg -例如：在url中指定了password为taosdata，在Properties中指定了password为taosdemo，那么，JDBC会使用url中的password建立连接。 + +例如：在 url 中指定了 password 为 taosdata，在 Properties 中指定了 password 为 taosdemo，那么，JDBC 会使用 url 中的 password 建立连接。 > 更多详细配置请参考[客户端配置][13] @@ -176,6 +230,7 @@ stmt.executeUpdate("use db"); // create table stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)"); ``` + > 注意：如果不使用 `use db` 指定数据库，则后续对表的操作都需要增加数据库名称作为前缀，如 db.tb。 ### 插入数据 @@ -186,6 +241,7 @@ int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now System.out.println("insert " + affectedRows + " rows."); ``` + > now 为系统内部函数，默认为服务器当前时间。 > `now + 1s` 代表服务器当前时间往后加 1 秒，数字后面代表时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。 @@ -207,8 +263,27 @@ while(resultSet.next()){ System.out.printf("%s, %d, %s\n", ts, temperature, humidity); } ``` + > 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。 +### 处理异常 +在报错后，通过SQLException可以获取到错误的信息和错误码： +```java +try (Statement statement = connection.createStatement()) { + // executeQuery + ResultSet resultSet = statement.executeQuery(sql); + // print result + printResult(resultSet); +} catch (SQLException e) { + System.out.println("ERROR Message: " + e.getMessage()); + System.out.println("ERROR Code: " + e.getErrorCode()); + e.printStackTrace(); +} +``` +JDBC连接器可能报错的错误码包括3种：JDBC driver本身的报错（错误码在0x2301到0x2350之间），JNI方法的报错（错误码在0x2351到0x2400之间），TDengine其他功能模块的报错。 +具体的错误码请参考： +* https://github.com/taosdata/TDengine/blob/develop/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +* https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h ### 订阅 @@ -224,7 +299,7 @@ TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from met * sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据 * restart：如果订阅已经存在，是重新开始，还是继续之前的订阅 -如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic' 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。 +如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。 #### 消费数据 @@ -242,7 +317,7 @@ while(true) { } ``` -`consume` 方法返回一个结果集，其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率（如例子中的`Thread.sleep(1000)`），否则会给服务端造成不必要的压力。 +`consume` 方法返回一个结果集，其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率（如例子中的 `Thread.sleep(1000)`），否则会给服务端造成不必要的压力。 #### 关闭订阅 @@ -252,7 +327,6 @@ sub.close(true); `close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。 - ### 关闭资源 ```java @@ -260,13 +334,17 @@ resultSet.close(); stmt.close(); conn.close(); ``` -> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。 + +> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。 + + ## 与连接池使用 **HikariCP** * 引入相应 HikariCP maven 依赖： + ```xml com.zaxxer @@ -276,31 +354,34 @@ conn.close(); ``` * 使用示例如下： + ```java public static void main(String[] args) throws SQLException { HikariConfig config = new HikariConfig(); + // jdbc properties config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log"); config.setUsername("root"); config.setPassword("taosdata"); - - config.setMinimumIdle(3); //minimum number of idle connection + // connection pool configurations + config.setMinimumIdle(10); //minimum number of idle connection config.setMaximumPoolSize(10); //maximum number of connection in the pool - config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool - config.setIdleTimeout(60000); // max idle time for recycle idle connection - config.setConnectionTestQuery("describe log.dn"); //validation query - config.setValidationTimeout(3000); //validation query timeout + config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setMaxLifetime(0); // maximum life time for each connection + config.setIdleTimeout(0); // max idle time for recycle idle connection + config.setConnectionTestQuery("select server_status()"); //validation query HikariDataSource ds = new HikariDataSource(config); //create datasource - + Connection connection = ds.getConnection(); // get connection Statement statement = connection.createStatement(); // get statement - - //query or insert + + //query or insert // ... - + connection.close(); // put back to conneciton pool } ``` + > 通过 HikariDataSource.getConnection() 获取连接后，使用完成后需要调用 close() 方法，实际上它并不会关闭连接，只是放回连接池中。 > 更多 HikariCP 使用问题请查看[官方说明][5] @@ -317,40 +398,32 @@ conn.close(); ``` * 使用示例如下： + ```java public static void main(String[] args) throws Exception { - Properties properties = new Properties(); - properties.put("driverClassName","com.taosdata.jdbc.TSDBDriver"); - properties.put("url","jdbc:TAOS://127.0.0.1:6030/log"); - properties.put("username","root"); - properties.put("password","taosdata"); - - properties.put("maxActive","10"); //maximum number of connection in the pool - properties.put("initialSize","3");//initial number of connection - properties.put("maxWait","10000");//maximum wait milliseconds for get connection from pool - properties.put("minIdle","3");//minimum number of connection in the pool - - properties.put("timeBetweenEvictionRunsMillis","3000");// the interval milliseconds to test connection - - properties.put("minEvictableIdleTimeMillis","60000");//the minimum milliseconds to keep idle - properties.put("maxEvictableIdleTimeMillis","90000");//the maximum milliseconds to keep idle - - properties.put("validationQuery","describe log.dn"); //validation query - properties.put("testWhileIdle","true"); // test connection while idle - properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true - properties.put("testOnReturn","false"); // don't need while testWhileIdle is true - - //create druid datasource - DataSource ds = DruidDataSourceFactory.createDataSource(properties); - Connection connection = ds.getConnection(); // get connection - Statement statement = connection.createStatement(); // get statement + DruidDataSource dataSource = new DruidDataSource(); + // jdbc properties + dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setUrl(url); + dataSource.setUsername("root"); + dataSource.setPassword("taosdata"); + // pool configurations + dataSource.setInitialSize(10); + dataSource.setMinIdle(10); + dataSource.setMaxActive(10); + dataSource.setMaxWait(30000); + dataSource.setValidationQuery("select server_status()"); + + Connection connection = dataSource.getConnection(); // get connection + Statement statement = connection.createStatement(); // get statement //query or insert // ... connection.close(); // put back to conneciton pool } ``` + > 更多 druid 使用问题请查看[官方说明][6] **注意事项** @@ -365,23 +438,56 @@ server_status()| Query OK, 1 row(s) in set (0.000141s) ``` + + ## 与框架使用 * Spring JdbcTemplate 中使用 taos-jdbcdriver，可参考 [SpringJdbcTemplate][11] * Springboot + Mybatis 中使用，可参考 [springbootdemo][12] + + +## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 + +| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | +| -------------------- | ----------------- | -------- | +| 2.0.12 及以上 | 2.0.8.0 及以上 | 1.8.x | +| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | +| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | + + + +## TDengine DataType 和 Java DataType + +TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： + +| TDengine DataType | Java DataType | +| ----------------- | ------------------ | +| TIMESTAMP | java.sql.Timestamp | +| INT | java.lang.Integer | +| BIGINT | java.lang.Long | +| FLOAT | java.lang.Float | +| DOUBLE | java.lang.Double | +| SMALLINT, TINYINT | java.lang.Short | +| BOOL | java.lang.Boolean | +| BINARY, NCHAR | java.lang.String | + + + ## 常见问题 * java.lang.UnsatisfiedLinkError: no taos in java.library.path - + **原因**：程序没有找到依赖的本地函数库 taos。 - - **解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 ` ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 - + + **解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 + * java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform - + **原因**：目前 TDengine 只支持 64 位 JDK。 - + **解决方法**：重新安装 64 位 JDK。 * 其它问题请参考 [Issues][7] @@ -396,8 +502,9 @@ Query OK, 1 row(s) in set (0.000141s) [8]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver [9]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver [10]: https://maven.aliyun.com/mvn/search -[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate +[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate [12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo -[13]: https://www.taosdata.com/cn/documentation20/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE +[13]: https://www.taosdata.com/cn/documentation/administrator/#client [14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client -[15]: https://www.taosdata.com/cn/getting-started/#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B +[15]: https://www.taosdata.com/cn/getting-started/#%E5%AE%A2%E6%88%B7%E7%AB%AF + diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/cn/08.connector/docs.md similarity index 51% rename from documentation20/webdocs/markdowndocs/connector-ch.md rename to documentation20/cn/08.connector/docs.md index 69c560bbc48a49ac040c964baa0d0f20ef9ca879..fb50e20e513aaf9b3791989c76c36eaca8df8f2b 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/cn/08.connector/docs.md @@ -1,10 +1,150 @@ # 连接器 -TDengine提供了丰富的应用程序开发接口，其中包括C/C++、JAVA、Python、RESTful、Go等，便于用户快速开发应用。 +TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、Python、Go、Node.js、C# 、RESTful 等，便于用户快速开发应用。 -注意：所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。 +![image-connecotr](page://images/connector.png) -## C/C++ Connector +目前TDengine的连接器可支持的平台广泛，包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下： + +| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** | +| ----------- | --------------- | --------------- | --------------- | --------------- | --------- | --------- | --------------- | ---------------- | -------------- | +| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | +| **C/C++** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ | +| **JDBC** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ | +| **Python** | ● | ● | ● | ○ | ● | ● | ○ | -- | ○ | +| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | +| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | +| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- | +| **RESTful** | ● | ● | ● | ● | ● | ● | ○ | ○ | ○ | + +其中 ● 表示经过官方测试验证， ○ 表示非官方测试验证。 + +注意： + +* 在没有安装TDengine服务端软件的系统中使用连接器（除RESTful外）访问 TDengine 数据库，需要安装相应版本的客户端安装包来使应用驱动（Linux系统中文件名为libtaos.so，Windows系统中为taos.dll）被安装在系统中，否则会产生无法找到相应库文件的错误。 +* 所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。 +* 升级到TDengine到2.0.8.0版本的用户，必须更新JDBC连接TDengine必须升级taos-jdbcdriver到2.0.12及以上。 +* 无论选用何种编程语言的连接器，2.0 及以上版本的 TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池，以避免连接内的“USE statement”状态量在线程之间相互干扰（但连接的查询和写入操作都是线程安全的）。 + +## 安装连接器驱动步骤 + +服务器应该已经安装TDengine服务端安装包。连接器驱动安装步骤如下： + +**Linux** + +**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载** + +* X64硬件环境：TDengine-client-2.x.x.x-Linux-x64.tar.gz + +* ARM64硬件环境：TDengine-client-2.x.x.x-Linux-aarch64.tar.gz + +* ARM32硬件环境：TDengine-client-2.x.x.x-Linux-aarch32.tar.gz + +**2. 解压缩软件包** + +将软件包放置在当前用户可读写的任意目录下，然后执行下面的命令： + +`tar -xzvf TDengine-client-xxxxxxxxx.tar.gz` + +其中xxxxxxx需要替换为实际版本的字符串。 + +**3. 执行安装脚本** + +解压软件包之后，会在解压目录下看到以下文件(目录)： + +​ *install_client.sh*：安装脚本，用于应用驱动程序 +​ *taos.tar.gz*：应用驱动安装包 +​ *driver*：TDengine应用驱动driver +​ *connector*: 各种编程语言连接器（go/grafanaplugin/nodejs/python/JDBC） +​ *examples*: 各种编程语言的示例程序(c/C#/go/JDBC/matlab/python/R) + +运行install_client.sh进行安装 + +**4. 配置taos.cfg** + +编辑taos.cfg文件(默认路径/etc/taos/taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030 + +**提示： 如本机没有部署TDengine服务，仅安装了应用驱动，则taos.cfg中仅需配置firstEP，无需配置FQDN。** + +**Windows x64/x86** + +**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载 ：** + +* X64硬件环境：TDengine-client-2.X.X.X-Windows-x64.exe + +* X86硬件环境：TDengine-client-2.X.X.X-Windows-x86.exe + +**2. 执行安装程序，按提示选择默认值，完成安装** + +**3. 安装路径** + +默认安装路径为：C:\TDengine，其中包括以下文件(目录)： + +​ *taos.exe*：taos shell命令行程序 + +​ *cfg* : 配置文件目录 +​ *driver*: 应用驱动动态链接库 +​ *examples*: 示例程序 bash/C/C#/go/JDBC/Python/Node.js +​ *include*: 头文件 +​ *log* : 日志文件 +​ *unins000.exe*: 卸载程序 + +**4. 配置taos.cfg** + +编辑taos.cfg文件(默认路径C:\TDengine\cfg\taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030 + +**提示：** + +**1. 如利用FQDN连接服务器，必须确认本机网络环境DNS已配置好，或在hosts文件中添加FQDN寻址记录，如编辑C:\Windows\system32\drivers\etc\hosts，添加如下的记录：** **192.168.1.99 h1.taos.com** + +**2．卸载：运行unins000.exe可卸载TDengine应用驱动。** + +**安装验证** + +以上安装和配置完成后，并确认TDengine服务已经正常启动运行，此时可以执行taos客户端进行登录。 + +**Linux环境：** + +在linux shell下直接执行 taos，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下： + +```mysql +$ taos +Welcome to the TDengine shell from Linux, Client Version:2.0.5.0 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. +taos> show databases; +name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB)| blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | +========================================================================================================================================================================================================================= +test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16| 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | +log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1| 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | +Query OK, 2 row(s) in set (0.001198s) +taos> +``` + +**Windows（x64/x86）环境：** + +在cmd下进入到c:\TDengine目录下直接执行 taos.exe，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下： + +```mysql + C:\TDengine>taos + Welcome to the TDengine shell from Linux, Client Version:2.0.5.0 + Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | + =================================================================================================================================================================================================================================================================== + test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | + log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | + Query OK, 2 row(s) in set (0.045000s) + taos> +``` + +## C/C++ Connector + +**C/C++连接器支持的系统有**： + +| **CPU类型** | x64（64bit） | | | ARM64 | ARM32 | +| ------------ | ------------ | -------- | -------- | -------- | ---------- | +| **OS类型** | Linux | Win64 | Win32 | Linux | Linux | +| **支持与否** | **支持** | **支持** | **支持** | **支持** | **开发中** | C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 _taos.h_（安装后，位于 _/usr/local/taos/include_）： @@ -12,10 +152,12 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine #include ``` -在编译时需要链接TDengine动态库 _libtaos.so_ （安装后，位于 _/usr/local/taos/driver_，gcc编译时，请加上 -ltaos）。 +注意： -如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 +* 在编译时需要链接TDengine动态库。Linux 为 *libtaos.so* ，安装后，位于 _/usr/local/taos/driver_。Windows为 taos.dll，安装后位于 *C:\TDengine*。 +* 如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 +使用C/C++连接器的示例代码请参见 https://github.com/taosdata/TDengine/tree/develop/tests/examples/c 。 ### 基础API @@ -25,129 +167,107 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine 初始化运行环境。如果应用没有主动调用该API，那么应用在调用`taos_connect`时将自动调用，故应用程序一般无需手动调用该API。 - - `void taos_cleanup()` 清理运行环境，应用退出前应调用此API。 - - `int taos_options(TSDB_OPTION option, const void * arg, ...)` 设置客户端选项，目前只支持时区设置（_TSDB_OPTION_TIMEZONE_）和编码设置（_TSDB_OPTION_LOCALE_）。时区和编码默认为操作系统当前设置。 - - `char *taos_get_client_info()` 获取客户端版本信息。 - -- `TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port)` +- `TAOS *taos_connect(const char *host, const char *user, const char *pass, const char *db, int port)` 创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含： - - ip：TDengine管理主节点的IP地址 + - host：TDengine管理主节点的FQDN - user：用户名 - pass：密码 - db：数据库名字，如果用户没有提供，也可以正常连接，用户可以通过该连接创建新的数据库，如果用户提供了数据库名字，则说明该数据库用户已经创建好，缺省使用该数据库 - port：端口号 - - 返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。 + 返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。 - `char *taos_get_server_info(TAOS *taos)` 获取服务端版本信息。 - - `int taos_select_db(TAOS *taos, const char *db)` 将当前的缺省数据库设置为`db`。 - - `void taos_close(TAOS *taos)` 关闭连接, 其中`taos`是`taos_connect`函数返回的指针。 - - ### 同步查询API 传统的数据库操作API，都属于同步操作。应用调用API后，一直处于阻塞状态，直到服务器返回结果。TDengine支持如下API： - `TAOS_RES* taos_query(TAOS *taos, const char *sql)` - 该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。返回值 NULL 表示失败。 - + 该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。不能通过返回值是否是 NULL 来判断执行结果是否失败，而是需要用`taos_errno`函数解析结果集中的错误代码来进行判断。 - `int taos_result_precision(TAOS_RES *res)` 返回结果集时间戳字段的精度，`0` 代表毫秒，`1` 代表微秒，`2` 代表纳秒。 - - `TAOS_ROW taos_fetch_row(TAOS_RES *res)` 按行获取查询结果集中的数据。 - - `int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows)` 批量获取查询结果集中的数据，返回值为获取到的数据的行数。 - - `int taos_num_fields(TAOS_RES *res)` 和 `int taos_field_count(TAOS_RES *res)` 这两个API等价，用于获取查询结果集中的列数。 - - `int* taos_fetch_lengths(TAOS_RES *res)` 获取结果集中每个字段的长度。 返回值是一个数组，其长度为结果集的列数。 - - `int taos_affected_rows(TAOS_RES *res)` 获取被所执行的 SQL 语句影响的行数。 - - `TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)` 获取查询结果集每列数据的属性（数据类型、名字、字节数），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下： - ```c - typedef struct taosField { - char name[65]; // 列名 - uint8_t type; // 数据类型 - int16_t bytes; // 字节数 - } TAOS_FIELD; - ``` - +```c +typedef struct taosField { + char name[65]; // 列名 + uint8_t type; // 数据类型 + int16_t bytes; // 字节数 +} TAOS_FIELD; +``` - `void taos_stop_query(TAOS_RES *res)` 停止一个查询的执行。 - - `void taos_free_result(TAOS_RES *res)` - 释放查询结果集以及相关的资源。查询完成后，务必调用该API释放资源，否则可能导致应用内存泄露。 - + 释放查询结果集以及相关的资源。查询完成后，务必调用该API释放资源，否则可能导致应用内存泄露。但也需注意，释放资源后，如果再调用`taos_consume`等获取查询结果的函数，将导致应用Crash。 - `char *taos_errstr(TAOS_RES *res)` 获取最近一次API调用失败的原因,返回值为字符串。 - - `char *taos_errno(TAOS_RES *res)` 获取最近一次API调用失败的原因，返回值为错误代码。 - -**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。 - +**注意**：2.0及以上版本 TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池。而不推荐在应用中将该连接 (TAOS\*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性，但 “USE statement” 等状态量有可能在线程之间相互干扰。此外，C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。 ### 异步查询API -同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2~4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。 +同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2～4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。 异步API都需要应用提供相应的回调函数，回调函数参数设置如下：前两个参数都是一致的，第三个参数依不同的API而定。第一个参数param是应用调用异步API时提供给系统的，用于回调时，应用能够找回具体操作的上下文，依具体实现而定。第二个参数是SQL操作的结果集，如果为空，比如insert操作，表示没有记录返回，如果不为空，比如select操作，表示有记录返回。 @@ -169,17 +289,8 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine * res：`taos_query_a`回调时返回的结果集 * fp：回调函数。其参数`param`是用户可定义的传递给回调函数的参数结构体；`numOfRows`是获取到的数据的行数（不是整个查询结果集的函数）。 在回调函数中，应用可以通过调用`taos_fetch_row`前向迭代获取批量记录中每一行记录。读完一块内的所有记录后，应用需要在回调函数中继续调用`taos_fetch_rows_a`获取下一批记录进行处理，直到返回的记录数（numOfRows）为零（结果返回完成）或记录数为负值（查询出错）。 - -- `void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);` - - 异步获取一条记录。其中： - - * res：`taos_query_a`回调时返回的结果集 - * fp：回调函数。其参数`param`是应用提供的一个用于回调的参数。回调时，第三个参数`row`指向一行记录。不同于`taos_fetch_rows_a`，应用无需调用`taos_fetch_row`来获取一行数据，更加简单，但数据提取性能不及批量获取的API。 - TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表，并可以同时对每张打开的表进行查询或者插入操作。需要指出的是，**客户端应用必须确保对同一张表的操作完全串行化**，即对同一个表的插入或查询操作未完成时（未返回时），不能够执行第二个插入或查询操作。 - ### 参数绑定API 除了直接调用 `taos_query` 进行查询，TDengine也提供了支持参数绑定的Prepare API，与 MySQL 一样，这些API目前也仅支持用问号`?`来代表待绑定的参数，具体如下： @@ -196,17 +307,17 @@ TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线 进行参数绑定，bind指向一个数组，需保证此数组的元素数量和顺序与sql语句中的参数完全一致。TAOS_BIND 的使用方法与 MySQL中的 MYSQL_BIND 一致，具体定义如下： - ```c - typedef struct TAOS_BIND { - int buffer_type; - void * buffer; - unsigned long buffer_length; // 未实际使用 - unsigned long *length; - int * is_null; - int is_unsigned; // 未实际使用 - int * error; // 未实际使用 - } TAOS_BIND; - ``` +```c +typedef struct TAOS_BIND { + int buffer_type; + void * buffer; + unsigned long buffer_length; // 未实际使用 + unsigned long *length; + int * is_null; + int is_unsigned; // 未实际使用 + int * error; // 未实际使用 +} TAOS_BIND; +``` - `int taos_stmt_add_batch(TAOS_STMT *stmt)` @@ -219,12 +330,11 @@ TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线 - `TAOS_RES* taos_stmt_use_result(TAOS_STMT *stmt)` 获取语句的结果集。结果集的使用方式与非参数化调用时一致，使用完成后，应对此结果集调用 `taos_free_result`以释放资源。 - + - `int taos_stmt_close(TAOS_STMT *stmt)` 执行完毕，释放所有资源。 - ### 连续查询接口 TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时间段，对一张或多张数据库的表(数据流)进行各种实时聚合计算操作。操作简单，仅有打开、关闭流的API。具体如下： @@ -238,14 +348,12 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * stime：是流式计算开始的时间，如果是0，表示从现在开始，如果不为零，表示从指定的时间开始计算（UTC时间从1970/1/1算起的毫秒数） * param：是应用提供的用于回调的一个参数，回调时，提供给应用 * callback: 第二个回调函数，会在连续查询自动停止时被调用。 - - 返回值为NULL，表示创建成功，返回值不为空，表示成功。 + 返回值为NULL，表示创建成功，返回值不为空，表示成功。 - `void taos_close_stream (TAOS_STREAM *tstr)` 关闭数据流，其中提供的参数是taos_open_stream的返回值。用户停止流式计算的时候，务必关闭该数据流。 - ### 数据订阅接口 @@ -270,7 +378,6 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * param：调用 `taos_subscribe`时客户程序提供的附加参数 * code：错误码 - * `TAOS_RES *taos_consume(TAOS_SUB *tsub)` 同步模式下，该函数用来获取订阅的结果。 用户应用程序将其置于一个循环之中。 如两次调用`taos_consume`的间隔小于订阅的轮询周期，API将会阻塞，直到时间间隔超过此周期。 如果数据库有新记录到达，该API将返回该最新的记录，否则返回一个没有记录的空结果集。 如果返回值为 `NULL`，说明系统出错。 异步模式下，用户程序不应调用此API。 @@ -279,13 +386,14 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 取消订阅。 如参数 `keepProgress` 不为0，API会保留订阅的进度信息，后续调用 `taos_subscribe` 时可以基于此进度继续；否则将删除进度信息，后续只能重新开始读取数据。 +## Python Connector -## Python Connector +Python连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1963.html) ### 安装准备 -* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端 [（Windows TDengine 客户端安装）][4] +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 * 已安装python 2.7 or >= 3.4 -* 已安装pip +* 已安装pip 或 pip3 ### Python客户端安装 @@ -297,21 +405,21 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 或 -​ `pip install src/connector/python/linux/python3/` +​ `pip3 install src/connector/python/linux/python3/` #### Windows 在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面 ```cmd cd C:\TDengine\connector\python\windows -pip install python2\ +python -m pip install python2\ ``` 或 ```cmd cd C:\TDengine\connector\python\windows -pip install python3\ +python -m pip install python3\ ``` -*如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。 +* 如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。 对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。 ### 使用 @@ -321,19 +429,21 @@ pip install python3\ * 导入TDengine客户端模块 ```python -import taos +import taos ``` * 获取连接并获取游标对象 + ```python conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") c1 = conn.cursor() ``` -*host 是TDengine 服务端所有IP, config 为客户端配置文件所在目录 +* host 是TDengine 服务端所有IP, config 为客户端配置文件所在目录 * 写入数据 + ```python import datetime - + # 创建数据库 c1.execute('create database db') c1.execute('use db') @@ -352,6 +462,7 @@ affected_rows = c1.execute(' '.join(sqlcmd)) ``` * 查询数据 + ```python c1.execute('select * from tb') # 拉取查询结果 @@ -361,7 +472,7 @@ numOfRows = c1.rowcount numOfCols = len(c1.description) for irow in range(numOfRows): print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])) - + # 直接使用cursor 循环拉取查询结果 c1.execute('select * from tb') for data in c1: @@ -369,6 +480,7 @@ for data in c1: ``` * 创建订阅 + ```python # 创建一个主题为 'test' 消费周期为1000毫秒的订阅 # 第一个参数为 True 表示重新开始订阅，如为 False 且之前创建过主题为 'test' 的订阅，则表示继续消费此订阅的数据，而不是重新开始消费所有数据 @@ -376,6 +488,7 @@ sub = conn.subscribe(True, "test", "select * from tb;", 1000) ``` * 消费订阅的数据 + ```python data = sub.consume() for d in data: @@ -383,36 +496,38 @@ for d in data: ``` * 取消订阅 + ```python sub.close() ``` - * 关闭连接 + ```python c1.close() conn.close() ``` + #### 帮助信息 用户可通过python的帮助信息直接查看模块的使用信息，或者参考tests/examples/python中的示例程序。以下为部分常用类和方法： -- _TDengineConnection_类 +- _TDengineConnection_ 类 参考python中help(taos.TDengineConnection)。 - 这个类对应客户端和TDengine建立的一个连接。在客户端多线程的场景下，这个连接实例可以是每个线程申请一个，也可以多线程共享一个连接。 + 这个类对应客户端和TDengine建立的一个连接。在客户端多线程的场景下，推荐每个线程申请一个独立的连接实例，而不建议多线程共享一个连接。 -- _TDengineCursor_类 +- _TDengineCursor_ 类 参考python中help(taos.TDengineCursor)。 这个类对应客户端进行的写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能夸线程共享使用，否则会导致返回结果出现错误。 -- _connect_方法 +- _connect_ 方法 用于生成taos.TDengineConnection的实例。 - ### Python客户端使用示例代码 + 在tests/examples/python中，我们提供了一个示例Python程序read_example.py，可以参考这个程序来设计用户自己的写入、查询程序。在安装了对应的客户端后，通过import taos引入taos类。主要步骤如下 - 通过taos.connect获取TDengineConnection对象，这个对象可以一个程序只申请一个，在多线程中共享。 @@ -422,9 +537,9 @@ conn.close() - 如果执行的是查询语句，则execute执行成功后，需要通过fetchall方法去拉取结果集。 具体方法可以参考示例代码。 -## RESTful Connector +## RESTful Connector -为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。 +为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。RESTful连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1965.html)。 ### HTTP请求格式 @@ -457,13 +572,13 @@ HTTP请求的BODY里就是一个完整的SQL语句，SQL语句中的数据表应 使用curl通过自定义身份认证方式来发起一个HTTP Request，语法如下： -``` +```bash curl -H 'Authorization: Basic ' -d '' :/rest/sql ``` 或者 -``` +```bash curl -u username:password -d '' :/rest/sql ``` @@ -473,10 +588,11 @@ curl -u username:password -d '' :/rest/sql 返回值为JSON格式，如下: -``` +```json { "status": "succ", - "head": ["Time Stamp","current", …], + "head": ["ts","current", …], + "column_meta": [["ts",9,8],["current",6,4], …], "data": [ ["2018-10-03 14:38:05.000", 10.3, …], ["2018-10-03 14:38:15.000", 12.6, …] @@ -487,16 +603,29 @@ curl -u username:password -d '' :/rest/sql 说明： -- status: 告知操作结果是成功还是失败 -- head: 表的定义，如果不返回结果集，仅有一列“affected_rows” -- data: 具体返回的数据，一排一排的呈现，如果不返回结果集，仅[[affected_rows]] -- rows: 表明总共多少行数据 +- status: 告知操作结果是成功还是失败。 +- head: 表的定义，如果不返回结果集，则仅有一列“affected_rows”。（从 2.0.17 版本开始，建议不要依赖 head 返回值来判断数据列类型，而推荐使用 column_meta。在未来版本中，有可能会从返回值中去掉 head 这一项。） +- column_meta: 从 2.0.17 版本开始，返回值中增加这一项来说明 data 里每一列的数据类型。具体每个列会用三个值来说明，分别为：列名、列类型、类型长度。例如`["current",6,4]`表示列名为“current”；列类型为 6，也即 float 类型；类型长度为 4，也即对应 4 个字节表示的 float。如果列类型为 binary 或 nchar，则类型长度表示该列最多可以保存的内容长度，而不是本次返回值中的具体数据长度。当列类型是 nchar 的时候，其类型长度表示可以保存的 unicode 字符数量，而不是 bytes。 +- data: 具体返回的数据，一行一行的呈现，如果不返回结果集，那么就仅有[[affected_rows]]。data 中每一行的数据列顺序，与 column_meta 中描述数据列的顺序完全一致。 +- rows: 表明总共多少行数据。 + +column_meta 中的列类型说明： +* 1：BOOL +* 2：TINYINT +* 3：SMALLINT +* 4：INT +* 5：BIGINT +* 6：FLOAT +* 7：DOUBLE +* 8：BINARY +* 9：TIMESTAMP +* 10：NCHAR ### 自定义授权码 HTTP请求中需要带有授权码``，用于身份识别。授权码通常由管理员提供，可简单的通过发送`HTTP GET`请求来获取授权码，操作如下： -``` +```bash curl http://:6041/rest/login// ``` @@ -510,13 +639,13 @@ curl http://:6041/rest/login// 获取授权码示例： -``` +```bash curl http://192.168.0.1:6041/rest/login/root/taosdata ``` 返回值： -``` +```json { "status": "succ", "code": 0, @@ -528,15 +657,16 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata - 在demo库里查询表d1001的所有记录： -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sql ``` 返回值： -``` +```json { "status": "succ", - "head": ["Time Stamp","current","voltage","phase"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ ["2018-10-03 14:38:05.000",10.3,219,0.31], ["2018-10-03 14:38:15.000",12.6,218,0.33] @@ -547,17 +677,18 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 - 创建库demo： -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6041/rest/sql ``` 返回值： -``` +```json { "status": "succ", "head": ["affected_rows"], + "column_meta": [["affected_rows",4,4]], "data": [[1]], - "rows": 1, + "rows": 1 } ``` @@ -567,16 +698,17 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19 HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sqlt ``` 返回值： -``` +```json { "status": "succ", - "head": ["column1","column2","column3"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ [1538548685000,10.3,219,0.31], [1538548695000,12.6,218,0.33] @@ -588,16 +720,17 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 #### 结果集采用UTC时间字符串 HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6041/rest/sqlutc ``` 返回值： -``` +```json { "status": "succ", - "head": ["column1","column2","column3"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ ["2018-10-03T14:38:05.000+0800",10.3,219,0.31], ["2018-10-03T14:38:15.000+0800",12.6,218,0.33] @@ -608,41 +741,49 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间 ### 重要配置项 -下面仅列出一些与RESTFul接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效 +下面仅列出一些与RESTful接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效 -- httpPort: 对外提供RESTFul服务的端口号，默认绑定到6041 -- httpMaxThreads: 启动的线程数量，默认为2 +- httpPort: 对外提供RESTful服务的端口号，默认绑定到6041 +- httpMaxThreads: 启动的线程数量，默认为2（2.0.17版本开始，默认值改为CPU核数的一半向下取整） - restfulRowLimit: 返回结果集（JSON格式）的最大条数，默认值为10240 - httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式 - httpDebugFlag: 日志开关，131：仅错误和报警信息，135：调试信息，143：非常详细的调试信息，默认131 -## CSharp Connector +## CSharp Connector -在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。 +C#连接器支持的系统有：Linux 64/Windows x64/Windows x86 -#### 安装TDengine客户端 +### 安装准备 -C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。 +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 +* .NET接口文件TDengineDrivercs.cs和参考程序示例TDengineTest.cs均位于Windows客户端install_directory/examples/C#目录下。 +* 在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。 -安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件 +### 安装验证 -- TDengineDriver.cs 调用taos.dll文件的Native C方法 -- TDengineTest.cs 参考程序示例 +运行install_directory/examples/C#/C#Checker/C#Checker.exe -在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件 +```cmd +cd {install_directory}/examples/C#/C#Checker +csc /optimize *.cs +C#Checker.exe -h +``` + +### C#连接器的使用 + +在Windows系统上，.NET应用程序可以使用TDengine的.NET接口来执行所有数据库的操作。使用.NET接口的步骤如下所示： -#### 使用方法 +1. 将.NET接口文件TDengineDrivercs.cs加入到应用程序所在.NET项目中。 +2. 用户可以参考TDengineTest.cs来定义数据库连接参数，以及如何执行数据插入、查询等操作； -- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中 -- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法 -- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中 +此.NET接口需要用到taos.dll文件，所以在执行应用程序前，拷贝Windows客户端install_directory/driver目录中的taos.dll文件到.NET项目最后生成.exe可执行文件所在文件夹。之后运行exe文件，即可访问TDengine数据库并做插入、查询等操作。 -#### 注意事项 +**注意：** -- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。 -- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。 +1. TDengine V2.0.3.0之后同时支持32位和64位Windows系统，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请选择对应的“X86” 或“x64”。 +2. 此.NET接口目前已经在Visual Studio 2015/2017中验证过，其它VS版本尚待验证。 -#### 第三方驱动 +### 第三方驱动 Maikebing.Data.Taos是一个TDengine的ADO.Net提供器，支持linux，windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考 @@ -653,10 +794,15 @@ https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos https://www.taosdata.com/blog/2020/11/02/1901.html ``` +## Go Connector + +### 安装准备 + +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 -## Go Connector +TDengine提供了GO驱动程序`taosSql`。 `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`。 -TDengine提供了GO驱动程序`taosSql`. `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go` +使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go 以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1951.html)。 ```Go import ( @@ -664,30 +810,67 @@ import ( _ "github.com/taosdata/driver-go/taosSql" ) ``` +**建议使用Go版本1.13或以上，并开启模块支持：** + +```bash +go env -w GO111MODULE=on +go env -w GOPROXY=https://goproxy.io,direct +``` + ### 常用API -* `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` - - 该API用来打开DB，返回一个类型为*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine - - **注意**： 该API成功创建的时候，并没有做权限等检查，只有在真正执行Query或者Exec的时候才能真正的去创建连接，并同时检查user/password/host/port是不是合法。 另外，由于整个驱动程序大部分实现都下沉到taosSql所依赖的libtaos中。所以，sql.Open本身特别轻量。 - -* `func (db *DB) Exec(query string, args ...interface{}) (Result, error)` + +- `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` + + 该API用来打开DB，返回一个类型为\*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine + + **注意**： 该API成功创建的时候，并没有做权限等检查，只有在真正执行Query或者Exec的时候才能真正的去创建连接，并同时检查user/password/host/port是不是合法。 另外，由于整个驱动程序大部分实现都下沉到taosSql所依赖的libtaos中。所以，sql.Open本身特别轻量。 + +- `func (db *DB) Exec(query string, args ...interface{}) (Result, error)` sql.Open内置的方法，用来执行非查询相关SQL - - -* `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)` - + +- `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)` + sql.Open内置的方法，用来执行查询语句 - - -## Node.js Connector -TDengine 同时也提供了node.js 的连接器。用户可以通过[npm](https://www.npmjs.com/)来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。[具体安装步骤如下](https://github.com/taosdata/tdengine/tree/master/src/connector/nodejs)： +- `func (db *DB) Prepare(query string) (*Stmt, error)` + + sql.Open内置的方法，Prepare creates a prepared statement for later queries or executions. + +- `func (s *Stmt) Exec(args ...interface{}) (Result, error)` + + sql.Open内置的方法，executes a prepared statement with the given arguments and returns a Result summarizing the effect of the statement. + +- `func (s *Stmt) Query(args ...interface{}) (*Rows, error)` + + sql.Open内置的方法，Query executes a prepared query statement with the given arguments and returns the query results as a \*Rows. + +- `func (s *Stmt) Close() error` + + sql.Open内置的方法，Close closes the statement. + +## Node.js Connector + +Node.js连接器支持的系统有： + +| **CPU类型** | x64（64bit） | | | aarch64 | aarch32 | +| ------------ | ------------ | -------- | -------- | -------- | -------- | +| **OS类型** | Linux | Win64 | Win32 | Linux | Linux | +| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** | + +Node.js连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1957.html) + +### 安装准备 + +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 + +### 安装Node.js连接器 + +用户可以通过[npm](https://www.npmjs.com/)来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。具体安装步骤如下： 首先，通过[npm](https://www.npmjs.com/)安装node.js 连接器. -```cmd +```bash npm install td2.0-connector ``` 我们建议用户使用npm 安装node.js连接器。如果您没有安装npm, 可以将*src/connector/nodejs/*拷贝到您的nodejs 项目目录下 @@ -697,62 +880,60 @@ npm install td2.0-connector ### Linux - `python` (建议`v2.7` , `v3.x.x` 目前还不支持) -- `node` 必须采用v10.x版本，其他版本存在包兼容性的问题。 +- `node` 2.0.6支持v12.x和v10.x，2.0.5及更早版本支持v10.x版本，其他版本可能存在包兼容性的问题。 - `make` - c语言编译器比如[GCC](https://gcc.gnu.org) -### macOS +### Windows -- `python` (建议`v2.7` , `v3.x.x` 目前还不支持) +#### 安装方法1 -- Xcode +使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具 - - 然后通过Xcode安装 +#### 安装方法2 - ``` - Command Line Tools - ``` +手动安装以下工具: - 在 - ``` - Xcode -> Preferences -> Locations - ``` +- 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) +- 安装 [Python](https://www.python.org/downloads/) 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` +- 进入`cmd`命令行界面, `npm config set msvs_version 2017` - 目录下可以找到这个工具。或者在终端里执行 +如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules) - ``` - xcode-select --install - ``` +如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64". +### 示例程序 - - 该步执行后 `gcc` 和 `make`就被安装上了 +示例程序源码位于install_directory/examples/nodejs，有： -### Windows +Node-example.js node.js示例源程序 +Node-example-raw.js -#### 安装方法1 +### 安装验证 -使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具 +在安装好TDengine客户端后，使用nodejsChecker.js程序能够验证当前环境是否支持nodejs方式访问Tdengine。 -#### 安装方法2 +验证方法： -手动安装以下工具: +1. 新建安装验证目录，例如：`~/tdengine-test`，拷贝github上nodejsChecker.js源程序。下载地址：（https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js）。 -- 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) -- 安装 [Python 2.7](https://www.python.org/downloads/) (`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` -- 进入`cmd`命令行界面, `npm config set msvs_version 2017` +2. 在命令中执行以下命令： -如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules) +```bash +npm init -y +npm install td2.0-connector +node nodejsChecker.js host=localhost +``` -如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64". +3. 执行以上步骤后，在命令行会输出nodejs连接Tdengine实例，并执行简答插入和查询的结果。 -### 使用方法 +### Node.js连接器的使用 -(http://docs.taosdata.com/node) -以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[该文档](http://docs.taosdata.com/node) +以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[TDengine Node.js connector](http://docs.taosdata.com/node) -#### 连接 +#### 建立连接 -使用node.js连接器时，必须先require ```td2.0-connector```，然后使用 ```taos.connect``` 函数。```taos.connect``` 函数必须提供的参数是```host```，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化```cursor``` 来和TDengine服务端通信 +使用node.js连接器时，必须先require `td2.0-connector`，然后使用 `taos.connect` 函数。`taos.connect` 函数必须提供的参数是`host`，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化`cursor` 来和TDengine服务端通信 ```javascript const taos = require('td2.0-connector'); @@ -766,15 +947,35 @@ var cursor = conn.cursor(); // Initializing a new cursor conn.close(); ``` +#### 执行SQL和插入数据 + +对于DDL语句（例如create database、create table、use等），可以使用cursor的execute方法。代码如下： + +```js +cursor.execute('create database if not exists test;') +``` + +以上代码创建了一个名称为test的数据库。对于DDL语句，一般没有返回值，cursor的execute返回值为0。 + +对于Insert语句，代码如下： + +```js +var affectRows = cursor.execute('insert into test.weather values(now, 22.3, 34);') +``` + +execute方法的返回值为该语句影响的行数，上面的sql向test库的weather表中，插入了一条数据，则返回值affectRows为1。 + +TDengine目前还不支持update和delete语句。 + #### 查询 -可通过 ```cursor.query``` 函数来查询数据库。 +可通过 `cursor.query` 函数来查询数据库。 ```javascript var query = cursor.query('show databases;') ``` -查询的结果可以通过 ```query.execute()``` 函数获取并打印出来 +查询的结果可以通过 `query.execute()` 函数获取并打印出来 ```javascript var promise = query.execute(); @@ -782,7 +983,7 @@ promise.then(function(result) { result.pretty(); }); ``` -格式化查询语句还可以使用```query```的```bind```方法。如下面的示例：```query```会自动将提供的数值填入查询语句的```?```里。 +格式化查询语句还可以使用`query`的`bind`方法。如下面的示例：`query`会自动将提供的数值填入查询语句的`?`里。 ```javascript var query = cursor.query('select * from meterinfo.meters where ts <= ? and areaid = ?;').bind(new Date(), 5); @@ -790,8 +991,7 @@ query.execute().then(function(result) { result.pretty(); }) ``` -如果在```query```语句里提供第二个参数并设为```true```也可以立即获取查询结果。如下： - +如果在`query`语句里提供第二个参数并设为`true`也可以立即获取查询结果。如下： ```javascript var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true) @@ -800,6 +1000,7 @@ promise.then(function(result) { }) ``` #### 异步函数 + 异步查询数据库的操作和上面类似，只需要在`cursor.execute`, `TaosQuery.execute`等函数后面加上`_a`。 ```javascript var promise1 = cursor.query('select count(*), avg(v1), avg(v2) from meter1;').execute_a() @@ -812,12 +1013,8 @@ promise2.then(function(result) { }) ``` - ### 示例 -[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例 - -[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`. - -[4]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client +[node-example.js](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例 +[node-example-raw.js](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`. diff --git a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md b/documentation20/cn/09.connections/docs.md similarity index 82% rename from documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md rename to documentation20/cn/09.connections/docs.md index 4e55f3c5e8f12f3277898b1f825dca5e22b2e6d1..79380f3bbd9680120f63f89a0bfbe6f31f5c7a74 100644 --- a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md +++ b/documentation20/cn/09.connections/docs.md @@ -1,7 +1,7 @@ # 与其他工具的连接 -## Grafana +## Grafana TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/)快速集成搭建数据监测报警系统，整个过程无需任何代码开发，TDengine中数据表中内容可以在仪表盘(DashBoard)上进行可视化展现。 @@ -13,7 +13,11 @@ TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/) TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin目录下。 -以CentOS 7.2操作系统为例，将tdengine目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。 +以CentOS 7.2操作系统为例，将grafanaplugin目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。 + +```bash +sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/tdengine +``` ### 使用 Grafana @@ -21,15 +25,15 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 用户可以直接通过 localhost:3000 的网址，登录 Grafana 服务器(用户名/密码:admin/admin)，通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示： -![img](../assets/add_datasource1.jpg) +![img](page://images/connections/add_datasource1.jpg) 点击 `Add data source` 可进入新增数据源页面，在查询框中输入 TDengine 可选择添加，如下图所示： -![img](../assets/add_datasource2.jpg) +![img](page://images/connections/add_datasource2.jpg) 进入数据源配置页面，按照默认提示修改相应配置即可： -![img](../assets/add_datasource3.jpg) +![img](page://images/connections/add_datasource3.jpg) * Host： TDengine 集群的中任意一台服务器的 IP 地址与 TDengine RESTful 接口的端口号(6041)，默认 http://localhost:6041 * User：TDengine 用户名。 @@ -37,13 +41,13 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 点击 `Save & Test` 进行测试，成功会有如下提示： -![img](../assets/add_datasource4.jpg) +![img](page://images/connections/add_datasource4.jpg) #### 创建 Dashboard 回到主界面创建 Dashboard，点击 Add Query 进入面板查询页面： -![img](../assets/create_dashboard1.jpg) +![img](page://images/connections/create_dashboard1.jpg) 如上图所示，在 Query 中选中 `TDengine` 数据源，在下方查询框可输入相应 sql 进行查询，具体说明如下： @@ -54,7 +58,7 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 按照默认提示查询当前 TDengine 部署所在服务器指定间隔系统内存平均使用量如下： -![img](../assets/create_dashboard2.jpg) +![img](page://images/connections/create_dashboard2.jpg) > 关于如何使用Grafana创建相应的监测界面以及更多有关使用Grafana的信息，请参考Grafana官方的[文档](https://grafana.com/docs/)。 @@ -64,14 +68,14 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 点击左侧 `Import` 按钮，并上传 `tdengine-grafana.json` 文件： -![img](../assets/import_dashboard1.jpg) +![img](page://images/connections/import_dashboard1.jpg) 导入完成之后可看到如下效果： -![img](../assets/import_dashboard2.jpg) +![img](page://images/connections/import_dashboard2.jpg) -## Matlab +## Matlab MatLab可以通过安装包内提供的JDBC Driver直接连接到TDengine获取数据到本地工作空间。 @@ -82,12 +86,15 @@ MatLab的适配有下面几个步骤，下面以Windows10上适配MatLab2017a为 - 将TDengine安装包内的驱动程序JDBCDriver-1.0.0-dist.jar拷贝到${matlab_root}\MATLAB\R2017a\java\jar\toolbox - 将TDengine安装包内的taos.lib文件拷贝至${matlab_ root _dir}\MATLAB\R2017a\lib\win64 - 将新添加的驱动jar包加入MatLab的classpath。在${matlab_ root _dir}\MATLAB\R2017a\toolbox\local\classpath.txt文件中添加下面一行 - -​ `$matlabroot/java/jar/toolbox/JDBCDriver-1.0.0-dist.jar` - +​ +``` +$matlabroot/java/jar/toolbox/JDBCDriver-1.0.0-dist.jar +``` - 在${user_home}\AppData\Roaming\MathWorks\MATLAB\R2017a\下添加一个文件javalibrarypath.txt, 并在该文件中添加taos.dll的路径，比如您的taos.dll是在安装时拷贝到了C:\Windows\System32下，那么就应该在javalibrarypath.txt中添加如下一行： - -​ `C:\Windows\System32` +​ +``` +C:\Windows\System32 +``` ### 在MatLab中连接TDengine获取数据 @@ -95,23 +102,25 @@ MatLab的适配有下面几个步骤，下面以Windows10上适配MatLab2017a为 - 创建一个连接： - `conn = database(‘db’, ‘root’, ‘taosdata’, ‘com.taosdata.jdbc.TSDBDriver’, ‘jdbc:TSDB://127.0.0.1:0/’)` - +```matlab +conn = database(‘db’, ‘root’, ‘taosdata’, ‘com.taosdata.jdbc.TSDBDriver’, ‘jdbc:TSDB://127.0.0.1:0/’) +``` - 执行一次查询： - `sql0 = [‘select * from tb’]` - - `data = select(conn, sql0);` - +```matlab +sql0 = [‘select * from tb’] +data = select(conn, sql0); +``` - 插入一条记录: - `sql1 = [‘insert into tb values (now, 1)’]` - - `exec(conn, sql1)` +```matlab +sql1 = [‘insert into tb values (now, 1)’] +exec(conn, sql1) +``` 更多例子细节请参考安装包内examples\Matlab\TDengineDemo.m文件。 -## R +## R R语言支持通过JDBC接口来连接TDengine数据库。首先需要安装R语言的JDBC包。启动R语言环境，然后执行以下命令安装R语言的JDBC支持库： @@ -146,4 +155,3 @@ TDengine客户端暂不支持如下函数： - dbExistsTable(conn, "test")：是否存在表test - dbListTables(conn)：显示连接中的所有表 - diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/cn/10.cluster/docs.md similarity index 82% rename from documentation20/webdocs/markdowndocs/cluster-ch.md rename to documentation20/cn/10.cluster/docs.md index db479417c5a22acf26bc7a829083cd21fd7dffb1..6d7d68fe1b5adc3fd7896c4070a4979398553cb2 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/cn/10.cluster/docs.md @@ -1,17 +1,19 @@ # TDengine 集群安装、管理 -多个TDengine服务器，也就是多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。而且在安装集群之前，先请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验单节点功能。 +多个TDengine服务器，也就是多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。而且在安装集群之前，先请按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)一章安装并体验单节点功能。 集群的每个数据节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取（如何配置FQDN，请参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)）。端口是这个数据节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 -## 准备工作 +关于集群搭建请参考[视频教程](https://www.taosdata.com/blog/2020/11/11/1961.html)。 + +## 准备工作 **第零步**：规划集群所有物理节点的FQDN，将规划好的FQDN分别添加到每个物理节点的/etc/hostname；修改每个物理节点的/etc/hosts，将所有集群物理节点的IP与FQDN的对应添加好。【如部署了DNS，请联系网络管理员在DNS上做好相关配置】 -**第一步**：如果搭建集群的物理节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) -**注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/）； +**第一步**：如果搭建集群的物理节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) +**注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（`rm -rf /var/lib/taos/*`）； **注意2：**客户端也需要配置，确保它可以正确解析每个节点的FQDN配置，不管是通过DNS服务，还是 Host 文件。 **第二步**：建议关闭所有物理节点的防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； @@ -57,15 +59,15 @@ arbitrator ha.taosdata.com:6042 | 8 | charset | 字符集编码 | | 9 | balance | 是否启动负载均衡 | | 10 | maxTablesPerVnode | 每个vnode中能够创建的最大表个数 | -| 11 | maxVgroupsPerDb | 每个DB中 能够使用的最大vnode个数 | +| 11 | maxVgroupsPerDb | 每个DB中能够使用的最大vgroup个数 | -## 启动第一个数据节点 +## 启动第一个数据节点 -按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)里的指示，启动第一个数据节点，例如h1.taosdata.com，然后执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： +按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)里的指示，启动第一个数据节点，例如h1.taosdata.com，然后执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： - ``` +``` Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. @@ -76,15 +78,15 @@ taos> show dnodes; Query OK, 1 row(s) in set (0.006385s) taos> - ``` +``` 上述命令里，可以看到这个刚启动的这个数据节点的End Point是：h1.taos.com:6030，就是这个新集群的firstEP。 -## 启动后续数据节点 +## 启动后续数据节点 将后续的数据节点添加到现有集群，具体有以下几步： -1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法在每个物理节点启动taosd； +1. 按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)一章的方法在每个物理节点启动taosd； 2. 在第一个数据节点，使用CLI程序taos, 登录进TDengine系统, 执行命令: @@ -113,7 +115,7 @@ taos> - firstEp这个参数仅仅在该数据节点首次加入集群时有作用，加入集群后，该数据节点会保存最新的mnode的End Point列表，不再依赖这个参数。 - 两个没有配置firstEp参数的数据节点dnode启动后，会独立运行起来。这个时候，无法将其中一个数据节点加入到另外一个数据节点，形成集群。**无法将两个独立的集群合并成为新的集群**。 -## 数据节点管理 +## 数据节点管理 上面已经介绍如何从零开始搭建集群。集群组建完后，还可以随时添加新的数据节点进行扩容，或删除数据节点，并检查集群当前状态。 @@ -137,6 +139,16 @@ DROP DNODE "fqdn:port"; 其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号 +**【注意】** + + - 一个数据节点一旦被drop之后，不能重新加入集群。需要将此节点重新部署（清空数据文件夹）。集群在完成drop dnode操作之前，会将该dnode的数据迁移走。 + + - 请注意 drop dnode 和 停止taosd进程是两个不同的概念，不要混淆：因为删除dnode之前要执行迁移数据的操作，因此被删除的dnode必须保持在线状态。待删除操作结束之后，才能停止taosd进程。 + + - 一个数据节点被drop之后，其他节点都会感知到这个dnodeID的删除操作，任何集群中的节点都不会再接收此dnodeID的请求。 + + - dnodeID的是集群自动分配的，不得人工指定。它在生成时递增的，不会重复。 + ### 查看数据节点 执行CLI程序taos,使用root账号登录进TDengine系统，执行： @@ -157,7 +169,7 @@ SHOW DNODES; SHOW VGROUPS; ``` -## vnode的高可用性 +## vnode的高可用性 TDengine通过多副本的机制来提供系统的高可用性，包括vnode和mnode的高可用性。 @@ -173,7 +185,7 @@ CREATE DATABASE demo replica 3; 因为vnode的引入，无法简单的给出结论：“集群中过半数据节点dnode工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为3，只有三个dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 -## Mnode的高可用性 +## Mnode的高可用性 TDengine集群是由mnode (taosd的一个模块，管理节点) 负责管理的，为保证mnode的高可用，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，有效范围为1-3。为保证元数据的强一致性，mnode副本之间是通过同步的方式进行数据复制的。 @@ -190,7 +202,7 @@ SHOW MNODES; **注意：**一个TDengine高可用系统，无论是vnode还是mnode, 都必须配置多个副本。 -## 负载均衡 +## 负载均衡 有三种情况，将触发负载均衡，而且都无需人工干预。 @@ -202,17 +214,18 @@ SHOW MNODES; **【提示】负载均衡由参数balance控制，它决定是否启动自动负载均衡。** -## 数据节点离线处理 +## 数据节点离线处理 如果一个数据节点离线，TDengine集群将自动检测到。有如下两种情况： -- 该数据节点离线超过一定时间（taos.cfg里配置参数offlineThreshold控制时长)，系统将自动把该数据节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的数据节点重现上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 +- 该数据节点离线超过一定时间（taos.cfg里配置参数offlineThreshold控制时长)，系统将自动把该数据节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的数据节点重新上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 - 离线后，在offlineThreshold的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。 **注意：**如果一个虚拟节点组（包括mnode组）里所归属的每个数据节点都处于离线或unsynced状态，必须等该虚拟节点组里的所有数据节点都上线、都能交换状态信息后，才能选出Master，该虚拟节点组才能对外提供服务。比如整个集群有3个数据节点，副本数为3，如果3个数据节点都宕机，然后2个数据节点重启，是无法工作的，只有等3个数据节点都重启成功，才能对外服务。 -## Arbitrator的使用 +## Arbitrator的使用 + +如果副本数为偶数，当一个vnode group里一半vnode不工作时，是无法从中选出master的。同理，一半mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了Arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含Arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到Arbitrator，那么节点B就能正常工作。 -如果副本数为偶数，当一个vnode group里一半vnode不工作时，是无法从中选出master的。同理，一半mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 +TDengine提供一个执行程序，名为 tarbitrator，找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/)，在TDengine Arbitrator Linux一节中，选择适合的版本下载并安装。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为Arbitrator的End Point。如果该参数配置了，当副本数为偶数时，系统将自动连接配置的Arbitrator。如果副本数为奇数，即使配置了Arbitrator，系统也不会去建立连接。 -TDengine提供一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/)，在TDengine Arbitrator Linux一节中，选择适合的版本下载并安装。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。如果副本数为奇数，即使配置了arbitrator, 系统也不会去建立连接。 diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/cn/11.administrator/docs.md similarity index 71% rename from documentation20/webdocs/markdowndocs/administrator-ch.md rename to documentation20/cn/11.administrator/docs.md index f54c6b91a1f6b171676176615f5377dfdbe8ff95..86ad8e5bb91e8883b4be4a2bec7b5fb7dcb4c483 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/cn/11.administrator/docs.md @@ -1,18 +1,18 @@ # TDengine的运营与维护 -## 容量规划 +## 容量规划 使用TDengine来搭建一个物联网大数据平台，计算资源、存储资源需要根据业务场景进行规划。下面分别讨论系统运行所需要的内存、CPU以及硬盘空间。 ### 内存需求 -每个DB可以创建固定数目的vnode，默认与CPU核数相同，可通过maxVgroupsPerDb配置；每个vnode会占用固定大小的内存（大小与数据库的配置参数blocks和cache有关)；每个Table会占用与标签总长度有关的内存；此外，系统会有一些固定的内存开销。因此，每个DB需要的系统内存可通过如下公式计算： +每个DB可以创建固定数目的vgroup，默认与CPU核数相同，可通过maxVgroupsPerDb配置；vgroup中的每个副本会是一个vnode；每个vnode会占用固定大小的内存（大小与数据库的配置参数blocks和cache有关)；每个Table会占用与标签总长度有关的内存；此外，系统会有一些固定的内存开销。因此，每个DB需要的系统内存可通过如下公式计算： ``` Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb) ``` -示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。 +示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000\*(0.25+0.5)/1000 = 499M。 实际运行的系统往往会根据数据特点的不同，将数据存放在不同的DB里。因此做规划时，也需要考虑。 @@ -35,7 +35,7 @@ TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景 Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable ``` -示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 +示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15\*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 用户可以通过参数keep，设置数据在磁盘中的最大保存时长。为进一步减少存储成本，TDengine还提供多级存储，最冷的数据可以存放在最廉价的存储介质上，应用的访问不用做任何调整，只是读取速度降低了。 @@ -47,9 +47,9 @@ Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable 因为TDengine具有很好的水平扩展能力，根据总量，再根据单个物理机或虚拟机的资源，就可以轻松决定需要购置多少台物理机或虚拟机了。 -**立即计算CPU、内存、存储，请参见：资源估算方法** +**立即计算CPU、内存、存储，请参见：[资源估算方法](https://www.taosdata.com/config/config.html)** -## 容错和灾备 +## 容错和灾备 ### 容错 @@ -76,10 +76,16 @@ TDengine集群的节点数必须大于等于副本数，否则创建表时将报 当TDengine集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine企业版还可以将副本部署在不同机房，从而实现异地容灾。 -## 服务端配置 +## 服务端配置 TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修改配置参数，以满足不同场景的需求。配置文件的缺省位置在/etc/taos目录，可以通过taosd命令行执行参数-c指定配置文件目录。比如taosd -c /home/user来指定配置文件位于/home/user这个目录。 +另外可以使用 “-C” 显示当前服务器配置参数： + +``` +taosd -C +``` + 下面仅仅列出一些重要的配置参数，更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节，而且这些参数的缺省配置都是工作的，一般无需设置。**注意：配置修改后，需要重启*taosd*服务才能生效。** - firstEp: taosd启动时，主动连接的集群中首个dnode的end point, 默认值为localhost:6030。 @@ -96,30 +102,33 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 - stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。 +- queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘，再乘 170 。单位为 MB（2.0.15 以前的版本中，此参数的单位是字节）。 +- ratioOfQueryCores: 设置查询线程的最大数量。最小值0 表示只有1个查询线程；最大值2表示最大建立2倍CPU核数的查询线程。默认为1，表示最大和CPU核数相等的查询线程。该值可以为小数，即0.5表示最大建立CPU核数一半的查询线程。 -**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 +**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030到6042共13个端口，而且必须TCP和UDP都打开。 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： - days：一个数据文件存储数据的时间跨度，单位为天，默认值：10。 - keep：数据库中数据保留的天数，单位为天，默认值：3650。 -- minRows: 文件块中记录的最小条数，单位为条，默认值：100。 -- maxRows: 文件块中记录的最大条数，单位为条，默认值：4096。 -- comp: 文件压缩标志位，0：关闭，1:一阶段压缩，2:两阶段压缩。默认值：2。 -- walLevel：WAL级别。1：写wal, 但不执行fsync; 2：写wal, 而且执行fsync。默认值：1。 +- minRows：文件块中记录的最小条数，单位为条，默认值：100。 +- maxRows：文件块中记录的最大条数，单位为条，默认值：4096。 +- comp：文件压缩标志位，0：关闭；1：一阶段压缩；2：两阶段压缩。默认值：2。 +- walLevel：WAL级别。1：写wal，但不执行fsync；2：写wal, 而且执行fsync。默认值：1。 - fsync：当wal设置为2时，执行fsync的周期。设置为0，表示每次写入，立即执行fsync。单位为毫秒，默认值：3000。 -- cache: 内存块的大小，单位为兆字节（MB），默认值：16。 -- blocks: 每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。 +- cache：内存块的大小，单位为兆字节（MB），默认值：16。 +- blocks：每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。 - replica：副本个数，取值范围：1-3。单位为个，默认值：1 - precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms +- cacheLast：是否在内存中缓存子表 last_row，0：关闭；1：开启。默认值：0。（从 2.0.11 版本开始支持此参数） -对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： +对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： ``` - create database demo days 10 cache 32 blocks 8 replica 3; + create database demo days 10 cache 32 blocks 8 replica 3 update 1; ``` -该SQL创建了一个库demo, 每个数据文件存储10天数据，内存块为32兆字节，每个VNODE占用8个内存块，副本数为3，而其他参数与系统配置完全一致。 +该SQL创建了一个库demo, 每个数据文件存储10天数据，内存块为32兆字节，每个VNODE占用8个内存块，副本数为3，允许更新，而其他参数与系统配置完全一致。 TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数必须与已有集群的配置相同，否则不能成功加入到集群中。会进行校验的参数如下： @@ -129,7 +138,7 @@ TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数 - offlineThreshold: dnode离线阈值，超过该时间将导致该dnode从集群中删除。单位为秒，默认值：86400*10（即10天）。 - statusInterval: dnode向mnode报告状态时长。单位为秒，默认值：1。 - maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值：1000000。 -- maxVgroupsPerDb: 每个数据库中能够使用的最大vnode个数。 +- maxVgroupsPerDb: 每个数据库中能够使用的最大vgroup个数。 - arbitrator: 系统中裁决器的end point，缺省为空。 - timezone、locale、charset 的配置见客户端配置。 @@ -140,8 +149,8 @@ ALTER DNODE ``` - dnode_id: 可以通过SQL语句"SHOW DNODES"命令获取 -- config: 要调整的日志参数，在如下列表中取值 - > resetlog 截断旧日志文件，创建一个新日志文件 +- config: 要调整的日志参数，在如下列表中取值 + > resetlog 截断旧日志文件，创建一个新日志文件 > debugFlag < 131 | 135 | 143 > 设置debugFlag为131、135或者143 例如： @@ -149,27 +158,36 @@ ALTER DNODE alter dnode 1 debugFlag 135; ``` -## 客户端配置 +## 客户端配置 + +TDengine系统的前台交互客户端应用程序为taos，以及应用驱动，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见帮助信息 `taos --help`。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。 -TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。 +**2.0.10.0 之后版本支持命令行以下参数显示当前客户端参数的配置** + +```bash +taos -C 或 taos --dump-config +``` 客户端配置参数 - firstEp: taos启动时，主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。 + +- secondEp: taos 启动时，如果 firstEp 连不上，将尝试连接 secondEp。 + - locale 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置 - + TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供一种专门的字段类型nchar。写入nchar字段的数据将统一采用UCS4-LE格式进行编码并发送到服务器。需要注意的是，编码正确性是客户端来保证。因此，如果用户想要正常使用nchar字段来存储诸如中文、日文、韩文等非ASCII字符，需要正确设置客户端的编码格式。 客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为UTF-8，部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中，编码则是CP936。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证nchar中的数据正确转换为UCS4-LE编码格式。 - 在 Linux 中 locale 的命名规则为: <语言>_<地区>.<字符集编码> 如：zh_CN.UTF-8，zh代表中文，CN代表大陆地区，UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码，由于Windows使用的locale中不是POSIX标准的locale格式，因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。 + 在 Linux 中 locale 的命名规则为: <语言>\_<地区>.<字符集编码> 如：zh_CN.UTF-8，zh代表中文，CN代表大陆地区，UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码，由于Windows使用的locale中不是POSIX标准的locale格式，因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。 - charset 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置 - + 如果配置文件中不设置charset，在Linux系统中，taos在启动时候，自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则尝试读取charset配置，如果读取charset配置也失败，则中断启动过程。 在Linux系统中，locale信息包含了字符编码信息，因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如： @@ -211,63 +229,64 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 均是合法的设置东八区时区的格式。 时区的设置对于查询和写入SQL语句中非Unix时间戳的内容（时间戳字符串、关键词now的解析）产生影响。例如： - ``` + ```sql SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08'; ``` 在东八区，SQL语句等效于 - ``` + ```sql SELECT count(*) FROM table_name WHERE TS<1554955268000; ``` 在UTC时区，SQL语句等效于 - ``` + ```sql SELECT count(*) FROM table_name WHERE TS<1554984068000; ``` 为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。 启动taos时，也可以从命令行指定一个taosd实例的end point，否则就从taos.cfg读取。 - + - maxBinaryDisplayWidth Shell中binary 和 nchar字段的显示宽度上限，超过此限制的部分将被隐藏。默认值：30。可在 shell 中通过命令 set max_binary_display_width nn 动态修改此选项。 - -## 用户管理 +## 用户管理 系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下： -``` +```sql CREATE USER PASS <'password'>; ``` -创建用户，并指定用户名和密码，密码需要用单引号引起来,单引号为英文半角 +创建用户，并指定用户名和密码，密码需要用单引号引起来，单引号为英文半角 -``` +```sql DROP USER ; ``` 删除用户，限root用户使用 -``` +```sql ALTER USER PASS <'password'>; ``` -修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 +修改用户密码，为避免被转换为小写，密码需要用单引号引用，单引号为英文半角 -``` -ALTER USER PRIVILEGE ; +```sql +ALTER USER PRIVILEGE ; ``` -修改用户权限为：super/write/read，不需要添加单引号 +修改用户权限为：write 或 read，不需要添加单引号 -``` +说明：系统内共有 super/write/read 三种权限级别，但目前不允许通过 alter 指令把 super 权限赋予用户。 + +```mysql SHOW USERS; ``` -显示所有用户 - +显示所有用户 + **注意：**SQL 语法中，< >表示需要用户输入的部分，但请不要输入< >本身 -## 数据导入 +## 数据导入 TDengine提供多种方便的数据导入功能，一种按脚本文件导入，一种按数据文件导入，一种是taosdump工具导入本身导出的文件。 @@ -312,17 +331,16 @@ taos> DESCRIBE d1001 ``` 那么可以用如下命令导入数据 -``` +```mysql taos> insert into d1001 file '~/data.csv'; Query OK, 9 row(s) affected (0.004763s) ``` - **taosdump工具导入** -TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见博客：TDengine DUMP工具使用指南 +TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见博客：[TDengine DUMP工具使用指南](https://www.taosdata.com/blog/2020/03/09/1334.html) -## 数据导出 +## 数据导出 为方便数据导出，TDengine提供了两种导出方式，分别是按表导出和用taosdump导出。 @@ -330,7 +348,7 @@ TDengine提供了方便的数据库导入导出工具taosdump。用户可以将t 如果用户需要导出一个表或一个STable中的数据，可在shell中运行 -``` +```mysql select * from >> data.csv; ``` @@ -338,43 +356,43 @@ select * from >> data.csv; **用taosdump导出数据** -TDengine提供了方便的数据库导出工具taosdump。用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表,所有数据或一时间段的数据，甚至仅仅表的定义。具体使用方法，请参见博客：TDengine DUMP工具使用指南 +TDengine提供了方便的数据库导出工具taosdump。用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表,所有数据或一时间段的数据，甚至仅仅表的定义。具体使用方法，请参见博客：[TDengine DUMP工具使用指南](https://www.taosdata.com/blog/2020/03/09/1334.html) -## 系统连接、任务查询管理 +## 系统连接、任务查询管理 系统管理员可以从CLI查询系统的连接、正在进行的查询、流式计算，并且可以关闭连接、停止正在进行的查询和流式计算。CLI里SQL语法如下： -``` +```mysql SHOW CONNECTIONS; ``` 显示数据库的连接，其中一列显示ip:port, 为连接的IP地址和端口号。 -``` +```mysql KILL CONNECTION ; ``` 强制关闭数据库连接，其中的connection-id是SHOW CONNECTIONS中显示的第一列的数字。 -``` +```mysql SHOW QUERIES; ``` 显示数据查询，其中第一列显示的以冒号隔开的两个数字为query-id，为发起该query应用连接的connection-id和查询次数。 -``` +```mysql KILL QUERY ; ``` 强制关闭数据查询，其中query-id是SHOW QUERIES中显示的 connection-id:query-no字串，如“105:2”，拷贝粘贴即可。 -``` +```mysql SHOW STREAMS; ``` 显示流式计算，其中第一列显示的以冒号隔开的两个数字为stream-id, 为启动该stream应用连接的connection-id和发起stream的次数。 -``` +```mysql KILL STREAM ; ``` @@ -386,7 +404,7 @@ TDengine启动后，会自动创建一个监测数据库log，并自动将服务 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。 -## 文件目录结构 +## 文件目录结构 安装TDengine后，默认会在操作系统中生成下列目录或文件： @@ -412,4 +430,62 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 - +## TDengine参数限制与保留关键字 + +- 数据库名：不能包含“.”以及特殊字符，不能超过32个字符 +- 表名：不能包含“.”以及特殊字符，与所属数据库名一起，不能超过192个字符 +- 表的列名：不能包含特殊字符，不能超过64个字符 +- 数据库名、表名、列名，都不能以数字开头 +- 表的列数：不能超过1024列 +- 记录的最大长度：包括时间戳8 byte，不能超过16KB +- 单条SQL语句默认最大字符串长度：65480 byte +- 数据库副本数：不能超过3 +- 用户名：不能超过23个byte +- 用户密码：不能超过15个byte +- 标签(Tags)数量：不能超过128个 +- 标签的总长度：不能超过16Kbyte +- 记录条数：仅受存储空间限制 +- 表的个数：仅受节点个数限制 +- 库的个数：仅受节点个数限制 +- 单个库上虚拟节点个数：不能超过64个 + +目前TDengine有将近200个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable名、数据列名及标签列名等。这些关键字列表如下： + +| 关键字列表 | | | | | +| ---------- | ----------- | ------------ | ---------- | --------- | +| ABLOCKS | CONNECTION | GROUP | MINUS | SLASH | +| ABORT | CONNECTIONS | GT | MNODES | SLIDING | +| ACCOUNT | COPY | ID | MODULES | SMALLINT | +| ACCOUNTS | COUNT | IF | NCHAR | SPREAD | +| ADD | CREATE | IGNORE | NE | STABLE | +| AFTER | CTIME | IMMEDIATE | NONE | STABLES | +| ALL | DATABASE | IMPORT | NOT | STAR | +| ALTER | DATABASES | IN | NOTNULL | STATEMENT | +| AND | DAYS | INITIALLY | NOW | STDDEV | +| AS | DEFERRED | INSERT | OF | STREAM | +| ASC | DELIMITERS | INSTEAD | OFFSET | STREAMS | +| ATTACH | DESC | INTEGER | OR | STRING | +| AVG | DESCRIBE | INTERVAL | ORDER | SUM | +| BEFORE | DETACH | INTO | PASS | TABLE | +| BEGIN | DIFF | IP | PERCENTILE | TABLES | +| BETWEEN | DISTINCT | IS | PLUS | TAG | +| BIGINT | DIVIDE | ISNULL | PRAGMA | TAGS | +| BINARY | DNODE | JOIN | PREV | TBLOCKS | +| BITAND | DNODES | KEEP | PRIVILEGE | TBNAME | +| BITNOT | DOT | KEY | QUERIES | TIMES | +| BITOR | DOUBLE | KILL | QUERY | TIMESTAMP | +| BOOL | DROP | LAST | RAISE | TINYINT | +| BOTTOM | EACH | LE | REM | TOP | +| BY | END | LEASTSQUARES | REPLACE | TRIGGER | +| CACHE | EQ | LIKE | REPLICA | UMINUS | +| CASCADE | EXISTS | LIMIT | RESET | UPLUS | +| CHANGE | EXPLAIN | LINEAR | RESTRICT | USE | +| CLOG | FAIL | LOCAL | ROW | USER | +| CLUSTER | FILL | LP | ROWS | USERS | +| COLON | FIRST | LSHIFT | RP | USING | +| COLUMN | FLOAT | LT | RSHIFT | VALUES | +| COMMA | FOR | MATCH | SCORES | VARIABLE | +| COMP | FROM | MAX | SELECT | VGROUPS | +| CONCAT | GE | METRIC | SEMI | VIEW | +| CONFIGS | GLOB | METRICS | SET | WAVG | +| CONFLICT | GRANTS | MIN | SHOW | WHERE | diff --git a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md b/documentation20/cn/12.taos-sql/01.error-code/docs.md similarity index 99% rename from documentation20/webdocs/markdowndocs/Taos Error Code-ch.md rename to documentation20/cn/12.taos-sql/01.error-code/docs.md index a9a3dee432c57b0078948d15799ac29df0167ae2..95975dba5aeeeee9f42c3bf0b34f48095ea83fa3 100644 --- a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md +++ b/documentation20/cn/12.taos-sql/01.error-code/docs.md @@ -1,6 +1,5 @@ # TDengine 2.0 错误码以及对应的十进制码 - | 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | |-----------------------| :---: | :---------: | :------------------------ | ---------------- | |TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647| diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/cn/12.taos-sql/docs.md similarity index 66% rename from documentation20/webdocs/markdowndocs/TAOS SQL-ch.md rename to documentation20/cn/12.taos-sql/docs.md index 760ebae4fc3b7ddd609d1bfb5689f51b05fc7cb7..aba8faa6771ad09a90b30b5bf3bb08664e8b6ca1 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -2,7 +2,7 @@ 本文档说明TAOS SQL支持的语法规则、主要查询功能、支持的SQL查询函数，以及常用技巧等内容。阅读本文档需要读者具有基本的SQL语言的基础。 -TAOS SQL是用户对TDengine进行数据写入和查询的主要工具。TAOS SQL为了便于用户快速上手，在一定程度上提供类似于标准SQL类似的风格和模式。严格意义上，TAOS SQL并不是也不试图提供SQL标准的语法。此外，由于TDengine针对的时序性结构化数据不提供修改和更新功能，因此在TAO SQL中不提供数据更新和数据删除的相关功能。 +TAOS SQL是用户对TDengine进行数据写入和查询的主要工具。TAOS SQL为了便于用户快速上手，在一定程度上提供类似于标准SQL类似的风格和模式。严格意义上，TAOS SQL并不是也不试图提供SQL标准的语法。此外，由于TDengine针对的时序性结构化数据不提供删除功能，因此在TAO SQL中不提供数据删除的相关功能。 本章节SQL语法遵循如下约定： @@ -25,7 +25,7 @@ taos> DESCRIBE meters; ``` 数据集包含4个智能电表的数据，按照TDengine的建模规则，对应4个子表，其名称分别是 d1001, d1002, d1003, d1004。 -## 支持的数据类型 +## 支持的数据类型 使用TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则： @@ -42,39 +42,52 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic | | 类型 | Bytes | 说明 | | ---- | :-------: | ------ | ------------------------------------------------------------ | | 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。 | -| 2 | INT | 4 | 整型，范围 [-2^31+1, 2^31-1], -2^31用作Null | -| 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63用于NULL | -| 4 | FLOAT | 4 | 浮点型，有效位数6-7，范围 [-3.4E38, 3.4E38] | -| 5 | DOUBLE | 8 | 双精度浮点型，有效位数15-16，范围 [-1.7E308, 1.7E308] | -| 6 | BINARY | 自定义 | 用于记录字符串，理论上，最长可以有16374字节，但由于每行数据最多16K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 | -| 7 | SMALLINT | 2 | 短整型， 范围 [-32767, 32767], -32768用于NULL | -| 8 | TINYINT | 1 | 单字节整型，范围 [-127, 127], -128用于NULL | -| 9 | BOOL | 1 | 布尔型，{true, false} | -| 10 | NCHAR | 自定义 | 用于记录非ASCII字符串，如中文字符。每个nchar字符占用4bytes的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 **\’**。nchar使用时须指定字符串大小，类型为nchar(10)的列表示此列的字符串最多存储10个nchar字符，会固定占用40bytes的空间。如用户字符串长度超出声明长度，则将会报错。 | - -**Tips**: TDengine对SQL语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。 - -## 数据库管理 +| 2 | INT | 4 | 整型，范围 [-2^31+1, 2^31-1], -2^31 用作 NULL | +| 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | +| 4 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | +| 5 | DOUBLE | 8 | 双精度浮点型，有效位数 15-16，范围 [-1.7E308, 1.7E308] | +| 6 | BINARY | 自定义 | 用于记录 ASCII 型字符串。理论上，最长可以有 16374 字节，但由于每行数据最多 16K 字节，实际上限一般小于理论值。 binary 仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如 binary(20) 定义了最长为 20 个字符的字符串，每个字符占 1 byte 的存储空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\’`。 | +| 7 | SMALLINT | 2 | 短整型， 范围 [-32767, 32767], -32768 用于 NULL | +| 8 | TINYINT | 1 | 单字节整型，范围 [-127, 127], -128 用于 NULL | +| 9 | BOOL | 1 | 布尔型，{true, false} | +| 10 | NCHAR | 自定义 | 用于记录非 ASCII 型字符串，如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小，类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符，会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度，将会报错。 | + +**Tips**: +1. TDengine 对 SQL 语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。 +2. 应避免使用 BINARY 类型来保存非 ASCII 型的字符串，会很容易导致数据乱码等错误。正确的做法是使用 NCHAR 类型来保存中文字符。 + +## 数据库管理 - **创建数据库** + ```mysql - CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep]; + CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [UPDATE 1]; ``` 说明： - - 1) `KEEP`是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据； - 2) 数据库名最大长度为33； - 3) 一条SQL 语句的最大长度为65480个字符； - 4) 数据库还有更多与存储相关的配置参数，请参见[系统管理](../administrator/#服务端配置)。 + + 1) KEEP是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据； + + 2) UPDATE 标志数据库支持更新相同时间戳数据； + + 3) 数据库名最大长度为33； + + 4) 一条SQL 语句的最大长度为65480个字符； + + 5) 数据库还有更多与存储相关的配置参数，请参见系统管理。 +- **显示系统当前参数** + + ```mysql + SHOW VARIABLES; + ``` - **使用数据库** + ```mysql USE db_name; ``` 使用/切换数据库 - - **删除数据库** ```mysql DROP DATABASE [IF EXISTS] db_name; @@ -85,49 +98,88 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ```mysql ALTER DATABASE db_name COMP 2; ``` - COMP参数是指修改数据库文件压缩标志位，取值范围为[0, 2]. 0表示不压缩，1表示一阶段压缩，2表示两阶段压缩。 + COMP 参数是指修改数据库文件压缩标志位，缺省值为 2，取值范围为 [0, 2]。0 表示不压缩，1 表示一阶段压缩，2 表示两阶段压缩。 ```mysql ALTER DATABASE db_name REPLICA 2; ``` - REPLICA参数是指修改数据库副本数，取值范围[1, 3]。在集群中使用，副本数必须小于或等于dnode的数目。 + REPLICA 参数是指修改数据库副本数，取值范围 [1, 3]。在集群中使用，副本数必须小于或等于 DNODE 的数目。 ```mysql ALTER DATABASE db_name KEEP 365; ``` - KEEP参数是指修改数据文件保存的天数，缺省值为3650，取值范围[days, 365000]，必须大于或等于days参数值。 + KEEP 参数是指修改数据文件保存的天数，缺省值为 3650，取值范围 [days, 365000]，必须大于或等于 days 参数值。 ```mysql ALTER DATABASE db_name QUORUM 2; ``` - QUORUM参数是指数据写入成功所需要的确认数。取值范围[1, 3]。对于异步复制，quorum设为1，具有master角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于2。原则上，Quorum >=1 并且 Quorum <= replica(副本数)，这个参数在启动一个同步模块实例时需要提供。 + QUORUM 参数是指数据写入成功所需要的确认数，取值范围 [1, 3]。对于异步复制，quorum 设为 1，具有 master 角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于 2。原则上，Quorum >= 1 并且 Quorum <= replica(副本数)，这个参数在启动一个同步模块实例时需要提供。 ```mysql ALTER DATABASE db_name BLOCKS 100; ``` - BLOCKS参数是每个VNODE (TSDB) 中有多少cache大小的内存块，因此一个VNODE的用的内存大小粗略为（cache * blocks）。取值范围[3, 1000]。 + BLOCKS 参数是每个 VNODE (TSDB) 中有多少 cache 大小的内存块，因此一个 VNODE 的用的内存大小粗略为（cache * blocks）。取值范围 [3, 1000]。 - **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。 + ```mysql + ALTER DATABASE db_name CACHELAST 0; + ``` + CACHELAST 参数控制是否在内存中缓存数据子表的 last_row。缺省值为 0，取值范围 [0, 1]。其中 0 表示不启用、1 表示启用。（从 2.0.11 版本开始支持） + **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。 - **显示系统所有数据库** + ```mysql SHOW DATABASES; ``` -## 表管理 +## 表管理 + - **创建数据表** - + ```mysql CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]); ``` 说明： + 1) 表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键； - 2) 表名最大长度为193； + + 2) 表名最大长度为192； + 3) 表的每行长度不能超过16k个字符; + 4) 子表名只能由字母、数字和下划线组成，且不能以数字开头 + 5) 使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节； +- **以超级表为模板创建数据表** + + ```mysql + CREATE TABLE [IF NOT EXISTS] tb_name USING stb_name TAGS (tag_value1, ...); + ``` + 以指定的超级表为模板，指定 tags 的值来创建数据表。 + +- **以超级表为模板创建数据表，并指定具体的 tags 列** + + ```mysql + CREATE TABLE [IF NOT EXISTS] tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...); + ``` + 以指定的超级表为模板，指定一部分 tags 列的值来创建数据表。（没被指定的 tags 列会设为空值。） + 说明：从 2.0.17 版本开始支持这种方式。在之前的版本中，不允许指定 tags 列，而必须显式给出所有 tags 列的取值。 + +- **批量创建数据表** + + ```mysql + CREATE TABLE [IF NOT EXISTS] tb_name1 USING stb_name TAGS (tag_value1, ...) tb_name2 USING stb_name TAGS (tag_value2, ...) ...; + ``` + 以更快的速度批量创建大量数据表。（服务器端 2.0.14 及以上版本） + + 说明： + + 1）批量建表方式要求数据表必须以超级表为模板。 + + 2）在不超出 SQL 语句长度限制的前提下，单条语句中的建表数量建议控制在 1000～3000 之间，将会获得比较理想的建表速度。 + - **删除数据表** ```mysql @@ -140,8 +192,17 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic SHOW TABLES [LIKE tb_name_wildcar]; ``` - 显示当前数据库下的所有数据表信息。说明：可在like中使用通配符进行名称的匹配。 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’_’下划线匹配一个字符。 + 显示当前数据库下的所有数据表信息。 + + 说明：可在like中使用通配符进行名称的匹配，这一通配符字符串最长不能超过24字节。 + + 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’\_’下划线匹配一个字符。 +- **在线修改显示字符宽度** + + ```mysql + SET MAX_BINARY_DISPLAY_WIDTH ; + ``` - **获取表的结构信息** @@ -155,8 +216,10 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ALTER TABLE tb_name ADD COLUMN field_name data_type; ``` 说明： + 1) 列的最大个数为1024，最小个数为2； - 2) 列名最大长度为65； + + 2) 列名最大长度为64； - **表删除列** @@ -165,71 +228,79 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ``` 如果表是通过[超级表](../super-table/)创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构 -## 超级表STable管理 +## 超级表STable管理 + +注意：在 2.0.15.0 及以后的版本中，开始支持 STABLE 保留字。也即，在本节后文的指令说明中，CREATE、DROP、ALTER 三个指令在老版本中保留字需写作 TABLE 而不是 STABLE。 + - **创建超级表** - + ```mysql - CREATE TABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); + CREATE STABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); ``` - 创建STable, 与创建表的SQL语法相似，但需指定TAGS字段的名称和类型 - + 创建 STable，与创建表的 SQL 语法相似，但需指定 TAGS 字段的名称和类型 + 说明： - 1) TAGS 列的数据类型不能是timestamp类型； + + 1) TAGS 列的数据类型不能是 timestamp 类型； + 2) TAGS 列名不能与其他列名相同； + 3) TAGS 列名不能为预留关键字； - 4) TAGS 最多允许128个，可以0个，总长度不超过16k个字符 - + + 4) TAGS 最多允许128个，至少1个，总长度不超过16k个字符。 + - **删除超级表** ```mysql - DROP TABLE [IF EXISTS] stb_name; + DROP STABLE [IF EXISTS] stb_name; ``` - 删除STable会自动删除通过STable创建的子表。 + 删除 STable 会自动删除通过 STable 创建的子表。 - **显示当前数据库下的所有超级表信息** ```mysql SHOW STABLES [LIKE tb_name_wildcar]; ``` - 查看数据库内全部STable，及其相关信息，包括STable的名称、创建时间、列数量、标签（TAG）数量、通过该STable建表的数量。 + 查看数据库内全部 STable，及其相关信息，包括 STable 的名称、创建时间、列数量、标签（TAG）数量、通过该 STable 建表的数量。 - **获取超级表的结构信息** ```mysql DESCRIBE stb_name; ``` - + - **超级表增加列** ```mysql - ALTER TABLE stb_name ADD COLUMN field_name data_type; + ALTER STABLE stb_name ADD COLUMN field_name data_type; ``` - **超级表删除列** ```mysql - ALTER TABLE stb_name DROP COLUMN field_name; + ALTER STABLE stb_name DROP COLUMN field_name; ``` -## 超级表 STable 中 TAG 管理 +## 超级表 STable 中 TAG 管理 + - **添加标签** - + ```mysql - ALTER TABLE stb_name ADD TAG new_tag_name tag_type; + ALTER STABLE stb_name ADD TAG new_tag_name tag_type; ``` - 为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符. + 为 STable 增加一个新的标签，并指定新标签的类型。标签总数不能超过 128 个，总长度不超过 16k 个字符。 - **删除标签** ```mysql - ALTER TABLE stb_name DROP TAG tag_name; + ALTER STABLE stb_name DROP TAG tag_name; ``` 删除超级表的一个标签，从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。 - **修改标签名** ```mysql - ALTER TABLE stb_name CHANGE TAG old_tag_name new_tag_name; + ALTER STABLE stb_name CHANGE TAG old_tag_name new_tag_name; ``` 修改超级表的标签名，从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。 @@ -238,9 +309,9 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ```mysql ALTER TABLE tb_name SET TAG tag_name=new_tag_value; ``` - 说明：除了更新标签的值的操作是针对子表进行，其他所有的标签操作（添加标签、删除标签等）均只能作用于STable，不能对单个子表操作。对STable添加标签以后，依托于该STable建立的所有表将自动增加了一个标签，所有新增标签的默认值都是NULL。 + 说明：除了更新标签的值的操作是针对子表进行，其他所有的标签操作（添加标签、删除标签等）均只能作用于 STable，不能对单个子表操作。对 STable 添加标签以后，依托于该 STable 建立的所有表将自动增加了一个标签，所有新增标签的默认值都是 NULL。 -## 数据写入 +## 数据写入 - **插入一条记录** ```mysql @@ -248,95 +319,104 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ``` 向表tb_name中插入一条记录 - - **插入一条记录，数据对应到指定的列** ```mysql - INSERT INTO tb_name (field1_name, ...) VALUES(field1_value, ...) + INSERT INTO tb_name (field1_name, ...) VALUES (field1_value1, ...); ``` 向表tb_name中插入一条记录，数据对应到指定的列。SQL语句中没有出现的列，数据库将自动填充为NULL。主键（时间戳）不能为NULL。 - - **插入多条记录** ```mysql - INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...)...; + INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` 向表tb_name中插入多条记录 - - **按指定的列插入多条记录** ```mysql - INSERT INTO tb_name (field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...) + INSERT INTO tb_name (field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` 向表tb_name中按指定的列插入多条记录 - - **向多个表插入多条记录** ```mysql - INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)... - tb2_name VALUES (field1_value1, ...)(field1_value2, ...)...; + INSERT INTO tb1_name VALUES (field1_value1, ...) (field1_value2, ...) ... + tb2_name VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` 同时向表tb1_name和tb2_name中分别插入多条记录 - - **同时向多个表按列插入多条记录** ```mysql - INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) - tb2_name (tb2_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...); + INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ... + tb2_name (tb2_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` 同时向表tb1_name和tb2_name中按列分别插入多条记录 - 注意： - 1) 如果时间戳为0，系统将自动使用服务器当前时间作为该记录的时间戳； - 2) 允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的keep值（数据保留的天数），允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的days值（数据文件存储数据的时间跨度，单位为天）。keep和days都是可以在创建数据库时指定的，缺省值分别是3650天和10天。 + 注意：允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的keep值（数据保留的天数），允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的days值（数据文件存储数据的时间跨度，单位为天）。keep和days都是可以在创建数据库时指定的，缺省值分别是3650天和10天。 -**历史记录写入**：可使用IMPORT或者INSERT命令，IMPORT的语法，功能与INSERT完全一样。 +- **插入记录时自动建表** + ```mysql + INSERT INTO tb_name USING stb_name TAGS (tag_value1, ...) VALUES (field_value1, ...); + ``` + 如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 tags 取值。 -## 数据查询 +- **插入记录时自动建表，并指定具体的 tags 列** + ```mysql + INSERT INTO tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...) VALUES (field_value1, ...); + ``` + 在自动建表时，可以只是指定部分 tags 列的取值，未被指定的 tags 列将取为空值。 -### 查询语法： +**历史记录写入**：可使用IMPORT或者INSERT命令，IMPORT的语法，功能与INSERT完全一样。 -```mysql -SELECT select_expr [, select_expr ...] - FROM {tb_name_list} - [WHERE where_condition] - [INTERVAL (interval_val [, interval_offset])] - [FILL fill_val] - [SLIDING fill_val] - [GROUP BY col_list] - [ORDER BY col_list { DESC | ASC }] - [SLIMIT limit_val [, SOFFSET offset_val]] - [LIMIT limit_val [, OFFSET offset_val]] - [>> export_file] -``` 说明：针对 insert 类型的 SQL 语句，我们采用的流式解析策略，在发现后面的错误之前，前面正确的部分SQL仍会执行。下面的sql中，insert语句是无效的，但是d1001仍会被创建。 + ```mysql -taos> create table meters(ts timestamp, current float, voltage int, phase float) tags(location binary(30), groupId int); +taos> CREATE TABLE meters(ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS(location BINARY(30), groupId INT); Query OK, 0 row(s) affected (0.008245s) -taos> show stables; +taos> SHOW STABLES; name | created_time | columns | tags | tables | ============================================================================================ meters | 2020-08-06 17:50:27.831 | 4 | 2 | 0 | Query OK, 1 row(s) in set (0.001029s) -taos> show tables; +taos> SHOW TABLES; Query OK, 0 row(s) in set (0.000946s) -taos> insert into d1001 using meters tags('Beijing.Chaoyang', 2); +taos> INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2); DB error: invalid SQL: keyword VALUES or FILE required -taos> show tables; +taos> SHOW TABLES; table_name | created_time | columns | stable_name | ====================================================================================================== d1001 | 2020-08-06 17:52:02.097 | 4 | meters | Query OK, 1 row(s) in set (0.001091s) ``` +## 数据查询 + +### 查询语法： + +```mysql +SELECT select_expr [, select_expr ...] + FROM {tb_name_list} + [WHERE where_condition] + [INTERVAL (interval_val [, interval_offset])] + [FILL fill_val] + [SLIDING fill_val] + [GROUP BY col_list] + [ORDER BY col_list { DESC | ASC }] + [SLIMIT limit_val [, SOFFSET offset_val]] + [LIMIT limit_val [, OFFSET offset_val]] + [>> export_file]; +``` + #### SELECT子句 + 一个选择子句可以是联合查询（UNION）和另一个查询的子查询（SUBQUERY）。 ##### 通配符 + 通配符 * 可以用于代指全部列。对于普通表，结果中只有普通列。 ```mysql taos> SELECT * FROM d1001; @@ -365,7 +445,6 @@ taos> SELECT * FROM meters; Query OK, 9 row(s) in set (0.002022s) ``` - 通配符支持表名前缀，以下两个SQL语句均为返回全部的列： ```mysql SELECT * FROM d1001; @@ -390,27 +469,52 @@ Query OK, 1 row(s) in set (0.020443s) 在使用SQL函数来进行查询过程中，部分SQL函数支持通配符操作。其中的区别在于： ```count(\*)```函数只返回一列。```first```、```last```、```last_row```函数则是返回全部列。 -``` -taos> select count(*) from d1001; +```mysql +taos> SELECT COUNT(*) FROM d1001; count(*) | ======================== 3 | Query OK, 1 row(s) in set (0.001035s) ``` -``` -taos> select first(*) from d1001; +```mysql +taos> SELECT FIRST(*) FROM d1001; first(ts) | first(current) | first(voltage) | first(phase) | ========================================================================================= 2018-10-03 14:38:05.000 | 10.30000 | 219 | 0.31000 | Query OK, 1 row(s) in set (0.000849s) ``` +##### 标签列 + +从 2.0.14 版本开始，支持在普通表的查询中指定 _标签列_，且标签列的值会与普通列的数据一起返回。 +```mysql +taos> SELECT location, groupid, current FROM d1001 LIMIT 2; + location | groupid | current | +====================================================================== + Beijing.Chaoyang | 2 | 10.30000 | + Beijing.Chaoyang | 2 | 12.60000 | +Query OK, 2 row(s) in set (0.003112s) +``` + +注意：普通表的通配符 * 中并不包含 _标签列_。 + +##### 获取标签列的去重取值 + +从 2.0.15 版本开始，支持在超级表查询标签列时，指定 distinct 关键字，这样将返回指定标签列的所有不重复取值。 +```mysql +SELECT DISTINCT tag_name FROM stb_name; +``` + +注意：目前 distinct 关键字只支持对超级表的标签列进行去重，而不能用于普通列。 + + + #### 结果集列名 ```SELECT```子句中，如果不指定返回结果集合的列名，结果集列名称默认使用```SELECT```子句中的表达式名称作为列名称。此外，用户可使用```AS```来重命名返回结果集合中列的名称。例如： -``` -taos> select ts, ts as primary_key_ts from d1001; +```mysql +taos> SELECT ts, ts AS primary_key_ts FROM d1001; ts | primary_key_ts | ==================================================== 2018-10-03 14:38:05.000 | 2018-10-03 14:38:05.000 | @@ -421,80 +525,84 @@ Query OK, 3 row(s) in set (0.001191s) 但是针对```first(*)```、```last(*)```、```last_row(*)```不支持针对单列的重命名。 #### 隐式结果列 + ```Select_exprs```可以是表所属列的列名，也可以是基于列的函数表达式或计算式，数量的上限256个。当用户使用了```interval```或```group by tags```的子句以后，在最后返回结果中会强制返回时间戳列（第一列）和group by子句中的标签列。后续的版本中可以支持关闭group by子句中隐式列的输出，列输出完全由select子句控制。 #### 表（超级表）列表 FROM关键字后面可以是若干个表（超级表）列表，也可以是子查询的结果。 如果没有指定用户的当前数据库，可以在表名称之前使用数据库的名称来指定表所属的数据库。例如：```power.d1001``` 方式来跨库使用表。 -``` +```mysql SELECT * FROM power.d1001; ------------------------------ -use power; +USE power; SELECT * FROM d1001; ``` #### 特殊功能 + 部分特殊的查询功能可以不使用FROM子句执行。获取当前所在的数据库 database() -``` -taos> SELECT database(); +```mysql +taos> SELECT DATABASE(); database() | ================================= power | Query OK, 1 row(s) in set (0.000079s) ``` 如果登录的时候没有指定默认数据库，且没有使用```use```命令切换数据，则返回NULL。 -``` -taos> SELECT database(); +```mysql +taos> SELECT DATABASE(); database() | ================================= NULL | Query OK, 1 row(s) in set (0.000184s) ``` 获取服务器和客户端版本号: -``` -taos> SELECT client_version(); +```mysql +taos> SELECT CLIENT_VERSION(); client_version() | =================== 2.0.0.0 | Query OK, 1 row(s) in set (0.000070s) -taos> SELECT server_version(); +taos> SELECT SERVER_VERSION(); server_version() | =================== 2.0.0.0 | Query OK, 1 row(s) in set (0.000077s) ``` 服务器状态检测语句。如果服务器正常，返回一个数字（例如 1）。如果服务器异常，返回error code。该SQL语法能兼容连接池对于TDengine状态的检查及第三方工具对于数据库服务器状态的检查。并可以避免出现使用了错误的心跳检测SQL语句导致的连接池连接丢失的问题。 -``` -taos> SELECT server_status(); +```mysql +taos> SELECT SERVER_STATUS(); server_status() | ================== 1 | Query OK, 1 row(s) in set (0.000074s) -taos> SELECT server_status() as status; +taos> SELECT SERVER_STATUS() AS status; status | ============== 1 | Query OK, 1 row(s) in set (0.000081s) ``` + #### TAOS SQL中特殊关键词 > TBNAME： 在超级表查询中可视为一个特殊的标签，代表查询涉及的子表名
    \_c0: 表示表（超级表）的第一列 #### 小技巧 + 获取一个超级表所有的子表名及相关的标签信息： -``` +```mysql SELECT TBNAME, location FROM meters; ``` 统计超级表下辖子表数量： -``` +```mysql SELECT COUNT(TBNAME) FROM meters; ``` 以上两个查询均只支持在Where条件子句中添加针对标签（TAGS）的过滤条件。例如： -``` +```mysql taos> SELECT TBNAME, location FROM meters; tbname | location | ================================================================== @@ -504,7 +612,7 @@ taos> SELECT TBNAME, location FROM meters; d1001 | Beijing.Chaoyang | Query OK, 4 row(s) in set (0.000881s) -taos> SELECT count(tbname) FROM meters WHERE groupId > 2; +taos> SELECT COUNT(tbname) FROM meters WHERE groupId > 2; count(tbname) | ======================== 2 | @@ -519,26 +627,28 @@ Query OK, 1 row(s) in set (0.001091s) ### 支持的条件过滤操作 -| Operation | Note | Applicable Data Types | -| --------- | ----------------------------- | ------------------------------------- | -| > | larger than | **`timestamp`** and all numeric types | -| < | smaller than | **`timestamp`** and all numeric types | -| >= | larger than or equal to | **`timestamp`** and all numeric types | -| <= | smaller than or equal to | **`timestamp`** and all numeric types | -| = | equal to | all types | -| <> | not equal to | all types | -| % | match with any char sequences | **`binary`** **`nchar`** | -| _ | match with a single char | **`binary`** **`nchar`** | - -1. 同时进行多个字段的范围过滤需要使用关键词AND进行连接不同的查询条件，暂不支持OR连接的不同列之间的查询过滤条件。 -2. 针对某一字段的过滤只支持单一时间区间过滤条件。但是针对其他的（普通）列或标签列，可以使用``` OR``` 条件进行组合条件的查询过滤。例如：((value > 20 and value < 30) OR (value < 12)) 。 +| Operation | Note | Applicable Data Types | +| ----------- | ----------------------------- | ------------------------------------- | +| > | larger than | **`timestamp`** and all numeric types | +| < | smaller than | **`timestamp`** and all numeric types | +| >= | larger than or equal to | **`timestamp`** and all numeric types | +| <= | smaller than or equal to | **`timestamp`** and all numeric types | +| = | equal to | all types | +| <> | not equal to | all types | +| between and | within a certain range | **`timestamp`** and all numeric types | +| % | match with any char sequences | **`binary`** **`nchar`** | +| _ | match with a single char | **`binary`** **`nchar`** | + +1. 同时进行多个字段的范围过滤，需要使用关键词 AND 来连接不同的查询条件，暂不支持 OR 连接的不同列之间的查询过滤条件。 +2. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如：((value > 20 AND value < 30) OR (value < 12)) 。 +3. 从 2.0.17 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。 ### SQL 示例 - 对于下面的例子，表tb1用以下语句创建 ```mysql - CREATE TABLE tb1 (ts timestamp, col1 int, col2 float, col3 binary(50)); + CREATE TABLE tb1 (ts TIMESTAMP, col1 INT, col2 FLOAT, col3 BINARY(50)); ``` - 查询tb1刚过去的一个小时的所有记录 @@ -556,7 +666,7 @@ Query OK, 1 row(s) in set (0.001091s) - 查询col1与col2的和，并取名complex, 时间大于2018-06-01 08:00:00.000, col2大于1.2，结果输出仅仅10条记录，从第5条开始 ```mysql - SELECT (col1 + col2) AS 'complex' FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' and col2 > 1.2 LIMIT 10 OFFSET 5; + SELECT (col1 + col2) AS 'complex' FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND col2 > 1.2 LIMIT 10 OFFSET 5; ``` - 查询过去10分钟的记录，col2的值大于3.14，并且将结果输出到文件 `/home/testoutpu.csv`. @@ -565,7 +675,7 @@ Query OK, 1 row(s) in set (0.001091s) SELECT COUNT(*) FROM tb1 WHERE ts >= NOW - 10m AND col2 > 3.14 >> /home/testoutpu.csv; ``` -## SQL 函数 +## SQL 函数 ### 聚合函数 @@ -575,37 +685,49 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表/超级表中记录行数或某列的非空值个数。 - 返回结果数据类型：长整型INT64。 - 应用字段：应用全部字段。 - 适用于：表、超级表。 - 说明：1）可以使用星号*来替代具体的字段，使用星号(*)返回全部记录数量。2）针对同一表的（不包含NULL值）字段查询结果均相同。3）如果统计对象是具体的列，则返回该列中非NULL值的记录数量。 + 功能说明：统计表/超级表中记录行数或某列的非空值个数。 + + 返回结果数据类型：长整型INT64。 + + 应用字段：应用全部字段。 + + 适用于：表、超级表。 + + 说明： + + 1）可以使用星号*来替代具体的字段，使用星号(*)返回全部记录数量。 + + 2）针对同一表的（不包含NULL值）字段查询结果均相同。 + + 3）如果统计对象是具体的列，则返回该列中非NULL值的记录数量。 示例： ```mysql - taos> SELECT COUNT(*), COUNT(VOLTAGE) FROM meters; + taos> SELECT COUNT(*), COUNT(voltage) FROM meters; count(*) | count(voltage) | ================================================ 9 | 9 | Query OK, 1 row(s) in set (0.004475s) - taos> SELECT COUNT(*), COUNT(VOLTAGE) FROM d1001; + taos> SELECT COUNT(*), COUNT(voltage) FROM d1001; count(*) | count(voltage) | ================================================ 3 | 3 | Query OK, 1 row(s) in set (0.001075s) ``` - - **AVG** ```mysql SELECT AVG(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的平均值。 - 返回结果数据类型：双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool字段。 - 适用于：表、超级表。 - + 功能说明：统计表/超级表中某列的平均值。 + + 返回结果数据类型：双精度浮点数Double。 + + 应用字段：不能应用在timestamp、binary、nchar、bool字段。 + + 适用于：表、超级表。 + 示例： ```mysql taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM meters; @@ -614,7 +736,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 11.466666751 | 220.444444444 | 0.293333333 | Query OK, 1 row(s) in set (0.004135s) - taos> SELECT AVG(current), AVG(voltage), AVG(phase) from d1001; + taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM d1001; avg(current) | avg(voltage) | avg(phase) | ==================================================================================== 11.733333588 | 219.333333333 | 0.316666673 | @@ -625,30 +747,35 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT TWA(field_name) FROM tb_name WHERE clause; ``` - 功能说明：时间加权平均函数。统计表/超级表中某列在一段时间内的时间加权平均。 - 返回结果数据类型：双精度浮点数Double。 + 功能说明：时间加权平均函数。统计表中某列在一段时间内的时间加权平均。 + + 返回结果数据类型：双精度浮点数Double。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 说明：时间加权平均(time weighted average, TWA）查询需要指定查询时间段的 _开始时间_ 和 _结束时间_ 。 - 适用于：表、超级表。 + + 适用于：表。 - **SUM** ```mysql SELECT SUM(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的和。 - 返回结果数据类型：双精度浮点数Double和长整型INT64。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 适用于：表、超级表。 + 功能说明：统计表/超级表中某列的和。 + + 返回结果数据类型：双精度浮点数Double和长整型INT64。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + + 适用于：表、超级表。 示例： ```mysql - taos> SELECT SUM(current), SUM(voltage), SUM(phase) from meters; + taos> SELECT SUM(current), SUM(voltage), SUM(phase) FROM meters; sum(current) | sum(voltage) | sum(phase) | ================================================================================ 103.200000763 | 1984 | 2.640000001 | Query OK, 1 row(s) in set (0.001702s) - taos> SELECT SUM(current), SUM(voltage), SUM(phase) from d1001; + taos> SELECT SUM(current), SUM(voltage), SUM(phase) FROM d1001; sum(current) | sum(voltage) | sum(phase) | ================================================================================ 35.200000763 | 658 | 0.950000018 | @@ -659,10 +786,13 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表中某列的均方差。 - 返回结果数据类型：双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 适用于：表。 + 功能说明：统计表中某列的均方差。 + + 返回结果数据类型：双精度浮点数Double。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + + 适用于：表。（从 2.0.15.1 版本开始，本函数也支持超级表） 示例： ```mysql @@ -677,10 +807,14 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val是自变量初始值，step_val是自变量的步长值。 - 返回结果数据类型：字符串表达式（斜率, 截距）。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 说明：自变量是时间戳，因变量是该列的值。 + 功能说明：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val是自变量初始值，step_val是自变量的步长值。 + + 返回结果数据类型：字符串表达式（斜率, 截距）。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + + 说明：自变量是时间戳，因变量是该列的值。 + 适用于：表。 示例： @@ -698,8 +832,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最小值。 - 返回结果数据类型：同应用的字段。 + 功能说明：统计表/超级表中某列的值最小值。 + + 返回结果数据类型：同应用的字段。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 示例： @@ -721,8 +857,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最大值。 - 返回结果数据类型：同应用的字段。 + 功能说明：统计表/超级表中某列的值最大值。 + + 返回结果数据类型：同应用的字段。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 示例： @@ -740,15 +878,23 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 1 row(s) in set (0.000987s) ``` - - **FIRST** ```mysql SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最先写入的非NULL值。 - 返回结果数据类型：同应用的字段。 - 应用字段：所有字段。 - 说明：1）如果要返回各个列的首个（时间戳最小）非NULL值，可以使用FIRST(*)；2) 如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；3) 如果结果集中所有列全部为NULL值，则不返回结果。 + 功能说明：统计表/超级表中某列的值最先写入的非NULL值。 + + 返回结果数据类型：同应用的字段。 + + 应用字段：所有字段。 + + 说明： + + 1）如果要返回各个列的首个（时间戳最小）非NULL值，可以使用FIRST(\*)； + + 2) 如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL； + + 3) 如果结果集中所有列全部为NULL值，则不返回结果。 示例： ```mysql @@ -769,10 +915,17 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最后写入的非NULL值。 - 返回结果数据类型：同应用的字段。 - 应用字段：所有字段。 - 说明：1）如果要返回各个列的最后（时间戳最大）一个非NULL值，可以使用LAST(*)；2）如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；如果结果集中所有列全部为NULL值，则不返回结果。 + 功能说明：统计表/超级表中某列的值最后写入的非NULL值。 + + 返回结果数据类型：同应用的字段。 + + 应用字段：所有字段。 + + 说明： + + 1）如果要返回各个列的最后（时间戳最大）一个非NULL值，可以使用LAST(\*)； + + 2）如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；如果结果集中所有列全部为NULL值，则不返回结果。 示例： ```mysql @@ -793,10 +946,17 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。 - 返回结果数据类型：同应用的字段。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。 + 功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。 + + 返回结果数据类型：同应用的字段。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + + 说明： + + 1）*k*值取值范围1≤*k*≤100； + + 2）系统同时返回该记录关联的时间戳列。 示例： ```mysql @@ -820,10 +980,17 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。 - 返回结果数据类型：同应用的字段。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。 + 功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。 + + 返回结果数据类型：同应用的字段。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + + 说明： + + 1）*k*值取值范围1≤*k*≤100； + + 2）系统同时返回该记录关联的时间戳列。 示例： ```mysql @@ -846,9 +1013,12 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; ``` - 功能说明：统计表中某列的值百分比分位数。 - 返回结果数据类型： 双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表中某列的值百分比分位数。 + + 返回结果数据类型： 双精度浮点数Double。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。 示例： @@ -864,10 +1034,14 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。 - 返回结果数据类型： 双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。 + + 返回结果数据类型： 双精度浮点数Double。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数 + ```mysql taos> SELECT APERCENTILE(current, 20) FROM d1001; apercentile(current, 20) | @@ -880,9 +1054,12 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; ``` - 功能说明：返回表（超级表）的最后一条记录。 - 返回结果数据类型：同应用的字段。 - 应用字段：所有字段。 + 功能说明：返回表（超级表）的最后一条记录。 + + 返回结果数据类型：同应用的字段。 + + 应用字段：所有字段。 + 说明：与last函数不同，last_row不支持时间范围限制，强制返回最后一条记录。 示例： @@ -905,11 +1082,14 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT DIFF(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表中某列的值与前一行对应值的差。 - 返回结果数据类型： 同应用字段。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表中某列的值与前一行对应值的差。 + + 返回结果数据类型： 同应用字段。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 说明：输出结果行数是范围内总行数减一，第一行没有结果输出。 - + 示例： ```mysql taos> SELECT DIFF(current) FROM d1001; @@ -920,14 +1100,16 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 2 row(s) in set (0.001162s) ``` - - **SPREAD** ```mysql SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的最大值和最小值之差。 - 返回结果数据类型： 双精度浮点数。 - 应用字段：不能应用在binary、nchar、bool类型字段。 + 功能说明：统计表/超级表中某列的最大值和最小值之差。 + + 返回结果数据类型： 双精度浮点数。 + + 应用字段：不能应用在binary、nchar、bool类型字段。 + 说明：可用于TIMESTAMP字段，此时表示记录的时间覆盖范围。 示例： @@ -945,16 +1127,22 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 1 row(s) in set (0.000836s) ``` - - **四则运算** ```mysql SELECT field_name [+|-|*|/|%][Value|field_name] FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列或多列间的值加、减、乘、除、取余计算结果。 - 返回结果数据类型：双精度浮点数。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 说明：1）支持两列或多列之间进行计算，可使用括号控制计算优先级;2）NULL字段不参与计算，如果参与计算的某行中包含NULL，该行的计算结果为NULL。 + 功能说明：统计表/超级表中某列或多列间的值加、减、乘、除、取余计算结果。 + + 返回结果数据类型：双精度浮点数。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + + 说明： + + 1）支持两列或多列之间进行计算，可使用括号控制计算优先级； + + 2）NULL字段不参与计算，如果参与计算的某行中包含NULL，该行的计算结果为NULL。 ```mysql taos> SELECT current + voltage * phase FROM d1001; @@ -966,16 +1154,17 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 3 row(s) in set (0.001046s) ``` -## 时间维度聚合 +## 时间维度聚合 + TDengine支持按时间段进行聚合，可以将表中数据按照时间段进行切割后聚合生成结果，比如温度传感器每秒采集一次数据，但需查询每隔10分钟的温度平均值。这个聚合适合于降维(down sample)操作, 语法如下： ```mysql -SELECT function_list FROM tb_name +SELECT function_list FROM tb_name [WHERE where_condition] INTERVAL (interval [, offset]) [FILL ({NONE | VALUE | PREV | NULL | LINEAR})] -SELECT function_list FROM stb_name +SELECT function_list FROM stb_name [WHERE where_condition] INTERVAL (interval [, offset]) [FILL ({ VALUE | PREV | NULL | LINEAR})] @@ -983,18 +1172,14 @@ SELECT function_list FROM stb_name ``` - 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。 -- WHERE语句可以指定查询的起止时间和其他过滤条件 +- WHERE语句可以指定查询的起止时间和其他过滤条件 - FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种： - 1. 不进行填充：NONE(默认填充模式)。 - - 2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。 - - 3. NULL填充：使用NULL填充数据。例如：fill(null)。 - - 4. PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。 - - -说明： + * 不进行填充：NONE(默认填充模式)。 + * VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。 + * NULL填充：使用NULL填充数据。例如：fill(null)。 + * PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。 + +说明： 1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。 2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。 @@ -1002,23 +1187,38 @@ SELECT function_list FROM stb_name **示例:** 智能电表的建表语句如下： ```mysql -CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int); +CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT); ``` 针对智能电表采集的数据，以10分钟为一个阶段，计算过去24小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非NULL值填充。 使用的查询语句如下： ```mysql -SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PERCENTILE(current, 50) FROM meters - WHERE TS>=NOW-1d +SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), PERCENTILE(current, 50) FROM meters + WHERE ts>=NOW-1d INTERVAL(10m) FILL(PREV); ``` -## TAOS SQL 边界限制 +## TAOS SQL 边界限制 + - 数据库名最大长度为32 - 表名最大长度为192，每行数据最大长度16k个字符 - 列名最大长度为64，最多允许1024列，最少需要2列，第一列必须是时间戳 -- 标签最多允许128个，可以0个，标签总长度不超过16k个字符 +- 标签最多允许128个，可以1个，标签总长度不超过16k个字符 - SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为1M - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 + +## TAOS SQL其他约定 + +**group by的限制** + +TAOS SQL支持对标签、tbname进行group by操作，也支持普通列进行group by，前提是：仅限一列且该列的唯一值小于10万个。 + +**join操作的限制** + +TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持两个表之间聚合后的四则运算。 + +**is not null与不为空的表达式适用范围** + +is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。 \ No newline at end of file diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/cn/13.faq/docs.md similarity index 71% rename from documentation20/webdocs/markdowndocs/faq-ch.md rename to documentation20/cn/13.faq/docs.md index 61f70c5962b4f83d720dfafd0e3e1759dad8dd25..727ccf653f50cb449c1f15e1c903706a2fab2068 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/cn/13.faq/docs.md @@ -1,5 +1,19 @@ # 常见问题 +## 0. 怎么报告问题？ + +如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包： +1. /var/log/taos （如果没有修改过默认路径） +2. /etc/taos + +附上必要的问题描述，包括使用的 TDengine 版本信息、平台环境信息、发生该问题的执行操作、出现问题的表征及大概的时间，在 [GitHub](https://github.com/taosdata/TDengine) 提交Issue。 + +为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。也可以通过如下SQL语句，临时设置taosd的日志级别。 +``` + alter dnode debugFlag 135; +``` +但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 + ## 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么？☆☆☆ 2.0版本在之前版本的基础上，进行了完全的重构，配置文件和数据文件是不兼容的。在升级之前务必进行如下操作： @@ -11,13 +25,16 @@ 5. 如果数据需要迁移数据或者数据文件损坏，请联系涛思数据官方技术支持团队，进行协助解决 ## 2. Windows平台下JDBCDriver找不到动态链接库，怎么办？ -请看为此问题撰写的技术博客 + +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/jdbcdriver找不到动态链接库/) ## 3. 创建数据表时提示more dnodes are needed -请看为此问题撰写的技术博客 + +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/创建数据表时提示more-dnodes-are-needed/) ## 4. 如何让TDengine crash时生成core文件？ -请看为此问题撰写的技术博客 + +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/06/tdengine-crash时生成core文件的方法/) ## 5. 遇到错误"Unable to establish connection", 我怎么办？ @@ -27,7 +44,7 @@ * 云服务器：检查云服务器的安全组是否打开TCP/UDP 端口6030-6042的访问权限 * 本地虚拟机：检查网络能否ping通，尽量避免使用`localhost` 作为hostname * 公司服务器：如果为NAT网络环境，请务必检查服务器能否将消息返回值客户端 - + 2. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用 3. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd* @@ -38,22 +55,20 @@ 6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的 -7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 +7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) 9. 如果仍不能排除连接故障 - + * Linux 系统请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` 检查服务器侧TCP端口连接是否工作：`nc -l {port}` 检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}` - - * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问 - -10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。 + * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问 +10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。 ## 6. 遇到错误“Unexpected generic error in RPC”或者"TDengine Error: Unable to resolve FQDN"， 我怎么办？ 产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： @@ -63,7 +78,6 @@ 3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件，查看该FQDN是否配置，并是否有正确的IP地址。 4. 如果网络配置OK，从客户端所在机器，你需要能Ping该连接的FQDN，否则客户端是无法连接服务器的 - ## 7. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。 @@ -74,29 +88,30 @@ TDengine还没有一组专用的validation queries。然而建议你使用系统 ## 9. 我可以删除或更新一条记录吗？ -不能。因为TDengine是为联网设备采集的数据设计的，不容许修改。但TDengine提供数据保留策略，只要数据记录超过保留时长，就会被自动删除。 +TDengine 目前尚不支持删除功能，未来根据用户需求可能会支持。 + +从 2.0.8.0 开始，TDengine 支持更新已经写入数据的功能。使用更新功能需要在创建数据库时使用 UPDATE 1 参数，之后可以使用 INSERT INTO 命令更新已经写入的相同时间戳数据。UPDATE 参数不支持 ALTER DATABASE 命令修改。没有使用 UPDATE 1 参数创建的数据库，写入相同时间戳的数据不会修改之前的数据，也不会报错。 ## 10. 我怎么创建超过1024列的表？ 使用2.0及其以上版本，默认支持1024列；2.0之前的版本，TDengine最大允许创建250列的表。但是如果确实超过限值，建议按照数据特性，逻辑地将这个宽表分解成几个小表。 -## 10. 最有效的写入数据的方法是什么？ +## 11. 最有效的写入数据的方法是什么？ 批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的多条记录。 -## 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ +## 12. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ -windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： +Windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： ```JAVA Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); Connection = DriverManager.getConnection(url, properties); ``` -## 12.TDengine GO windows驱动的如何编译？ -请看为此问题撰写的技术博客 ## 13.JDBC报错： the excuted SQL is not a DML or a DDL？ + 请更新至最新的JDBC驱动 ```JAVA @@ -105,28 +120,37 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 14. taos connect failed, reason: invalid timestamp - -常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。 +## 14. taos connect failed, reason: invalid timestamp +常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。 ## 15. 表名显示不全 由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。 +## 16. 如何进行数据迁移？ +TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A移动机器B时，注意如下两件事： -## 16. 怎么报告问题？ +- 2.0.0.0 至 2.0.6.x 的版本，重新配置机器B的hostname为机器A的hostname +- 2.0.7.0 及以后的版本，到/var/lib/taos/dnode下，修复dnodeEps.json的dnodeId对应的FQDN，重启。确保机器内所有机器的此文件是完全相同的。 +- 1.x 和 2.x 版本的存储结构不兼容，需要使用迁移工具或者自己开发应用导出导入数据。 -如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: -1. /var/log/taos -2. /etc/taos +## 17. 如何在命令行程序 taos 中临时调整日志级别 -附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 +为了调试方便，从 2.0.16 版本开始，命令行程序 taos 新增了与日志记录相关的两条指令： -为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。也可以通过如下SQL语句，临时设置taosd的日志级别。 +```mysql +ALTER LOCAL flag_name flag_value; ``` - alter dnode debugFlag 135; + +其含义是，在当前的命令行程序下，修改一个特定模块的日志记录级别（只对当前命令行程序有效，如果 taos 命令行程序重启，则需要重新设置）： +- flag_name 的取值可以是：debugFlag，cDebugFlag，tmrDebugFlag，uDebugFlag，rpcDebugFlag +- flag_value 的取值可以是：131（输出错误和警告日志），135（ 输出错误、警告和调试日志），143（ 输出错误、警告、调试和跟踪日志） + +```mysql +ALTER LOCAL RESETLOG; ``` -但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 + +其含义是，清空本机所有由客户端生成的日志文件。 diff --git a/documentation20/webdocs/assets/dnode.png b/documentation20/cn/images/architecture/dnode.png similarity index 100% rename from documentation20/webdocs/assets/dnode.png rename to documentation20/cn/images/architecture/dnode.png diff --git a/documentation20/webdocs/assets/message.png b/documentation20/cn/images/architecture/message.png similarity index 100% rename from documentation20/webdocs/assets/message.png rename to documentation20/cn/images/architecture/message.png diff --git a/documentation20/webdocs/assets/modules.png b/documentation20/cn/images/architecture/modules.png similarity index 100% rename from documentation20/webdocs/assets/modules.png rename to documentation20/cn/images/architecture/modules.png diff --git a/documentation20/cn/images/architecture/multi_tables.png b/documentation20/cn/images/architecture/multi_tables.png new file mode 100644 index 0000000000000000000000000000000000000000..0cefaab6a9a4cdd671c671f7c6186dea41415ff0 Binary files /dev/null and b/documentation20/cn/images/architecture/multi_tables.png differ diff --git a/documentation20/cn/images/architecture/replica-forward.png b/documentation20/cn/images/architecture/replica-forward.png new file mode 100644 index 0000000000000000000000000000000000000000..bf616e030b130603eceb5dccfd30b4a1dfa68ea5 Binary files /dev/null and b/documentation20/cn/images/architecture/replica-forward.png differ diff --git a/documentation20/webdocs/assets/replica-master.png b/documentation20/cn/images/architecture/replica-master.png similarity index 100% rename from documentation20/webdocs/assets/replica-master.png rename to documentation20/cn/images/architecture/replica-master.png diff --git a/documentation20/cn/images/architecture/replica-restore.png b/documentation20/cn/images/architecture/replica-restore.png new file mode 100644 index 0000000000000000000000000000000000000000..1558e5ed0108d23efdc6b5d9ea0e44a1dff45d28 Binary files /dev/null and b/documentation20/cn/images/architecture/replica-restore.png differ diff --git a/documentation20/cn/images/architecture/structure.png b/documentation20/cn/images/architecture/structure.png new file mode 100644 index 0000000000000000000000000000000000000000..4fc8f47ab0a30d95b85ba1d85105726ed981e56e Binary files /dev/null and b/documentation20/cn/images/architecture/structure.png differ diff --git a/documentation20/webdocs/assets/vnode.png b/documentation20/cn/images/architecture/vnode.png similarity index 100% rename from documentation20/webdocs/assets/vnode.png rename to documentation20/cn/images/architecture/vnode.png diff --git a/documentation20/webdocs/assets/write_master.png b/documentation20/cn/images/architecture/write_master.png similarity index 100% rename from documentation20/webdocs/assets/write_master.png rename to documentation20/cn/images/architecture/write_master.png diff --git a/documentation20/webdocs/assets/write_slave.png b/documentation20/cn/images/architecture/write_slave.png similarity index 100% rename from documentation20/webdocs/assets/write_slave.png rename to documentation20/cn/images/architecture/write_slave.png diff --git a/documentation20/cn/images/connections/add_datasource1.jpg b/documentation20/cn/images/connections/add_datasource1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1f0f5110f312c57f3ec1788bbc02f04fac6ac142 Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource1.jpg differ diff --git a/documentation20/cn/images/connections/add_datasource2.jpg b/documentation20/cn/images/connections/add_datasource2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fa7a83e00e96fae649910dff4edf5f5bdadd7850 Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource2.jpg differ diff --git a/documentation20/cn/images/connections/add_datasource3.jpg b/documentation20/cn/images/connections/add_datasource3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc850ad08ff1174de972906842e0d5ee64e6e5cb Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource3.jpg differ diff --git a/documentation20/cn/images/connections/add_datasource4.jpg b/documentation20/cn/images/connections/add_datasource4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3ba73e50d455111f8621f4165746078554c2d790 Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource4.jpg differ diff --git a/documentation20/cn/images/connections/create_dashboard1.jpg b/documentation20/cn/images/connections/create_dashboard1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b83c3a1714e9e7540e0b06239ef7c1c4f63fe2c Binary files /dev/null and b/documentation20/cn/images/connections/create_dashboard1.jpg differ diff --git a/documentation20/cn/images/connections/create_dashboard2.jpg b/documentation20/cn/images/connections/create_dashboard2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fe5d768ac55254251e0290bf257178f5ff28f5a5 Binary files /dev/null and b/documentation20/cn/images/connections/create_dashboard2.jpg differ diff --git a/documentation20/cn/images/connections/import_dashboard1.jpg b/documentation20/cn/images/connections/import_dashboard1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9d2ce7ed65eb0c2c729de50283b30491793493dc Binary files /dev/null and b/documentation20/cn/images/connections/import_dashboard1.jpg differ diff --git a/documentation20/cn/images/connections/import_dashboard2.jpg b/documentation20/cn/images/connections/import_dashboard2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..94b09f0ee39552bb84f7ba1f65815ce2c9548b2d Binary files /dev/null and b/documentation20/cn/images/connections/import_dashboard2.jpg differ diff --git a/documentation20/cn/images/connector.png b/documentation20/cn/images/connector.png new file mode 100644 index 0000000000000000000000000000000000000000..6030bd73f51123615eabacfa5d734918559ce3d9 Binary files /dev/null and b/documentation20/cn/images/connector.png differ diff --git a/documentation20/webdocs/assets/EcoSystem.png b/documentation20/cn/images/eco_system.png similarity index 100% rename from documentation20/webdocs/assets/EcoSystem.png rename to documentation20/cn/images/eco_system.png diff --git a/documentation20/cn/images/tdengine-jdbc-connector.png b/documentation20/cn/images/tdengine-jdbc-connector.png new file mode 100644 index 0000000000000000000000000000000000000000..fdf1dd3fcc5ee222c4a8753efa2c95c5257314bf Binary files /dev/null and b/documentation20/cn/images/tdengine-jdbc-connector.png differ diff --git a/documentation20/webdocs/assets/replica-forward.jpg b/documentation20/webdocs/assets/replica-forward.jpg deleted file mode 100644 index 00b8c357b9ca2f067127c7dfae37b386b268f8dd..0000000000000000000000000000000000000000 Binary files a/documentation20/webdocs/assets/replica-forward.jpg and /dev/null differ diff --git a/documentation20/webdocs/assets/replica-restore.jpg b/documentation20/webdocs/assets/replica-restore.jpg deleted file mode 100644 index 6da1bed4cc90b8a119b2fddf5f66ef3ab1b0a2fd..0000000000000000000000000000000000000000 Binary files a/documentation20/webdocs/assets/replica-restore.jpg and /dev/null differ diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md deleted file mode 100644 index f1f2d58f0562b831395e5177eb34ec3534aaa715..0000000000000000000000000000000000000000 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ /dev/null @@ -1,131 +0,0 @@ -# TDengine文档 - -TDengine是一个高效的存储、查询、分析时序大数据的平台，专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它，但建议您在使用前仔细阅读一遍下面的文档，特别是[数据模型](https://www.taosdata.com/cn/documentation20/data-model-and-architecture)与[数据建模](https://www.taosdata.com/cn/documentation20/model)一节。除本文档之外，欢迎[下载产品白皮书](https://www.taosdata.com/downloads/TDengine White Paper.pdf)。如需查阅TDengine 1.6 文档，请点击[这里](https://www.taosdata.com/cn/documentation16/)访问。 - -## TDengine介绍 - -- TDengine 简介及特色 -- TDengine 适用场景 - -## [立即开始](https://www.taosdata.com/cn/getting-started) - -- [快捷安装](https://www.taosdata.com/cn/documentation20/getting-started/#快捷安装)：可通过源码、安装包或docker安装，三秒钟搞定 -- [轻松启动](https://www.taosdata.com/cn/documentation20/getting-started/#轻松启动)：使用systemctl 启停TDengine -- [命令行程序TAOS](https://www.taosdata.com/cn/documentation20/getting-started/#TDengine命令行程序)：访问TDengine的简便方式 -- [极速体验](https://www.taosdata.com/cn/documentation20/getting-started/#TDengine-极速体验)：运行示例程序，快速体验高效的数据插入、查询 - -## [数据模型和整体架构](https://www.taosdata.com/cn/documentation20/architecture) - -- [数据模型](https://www.taosdata.com/cn/documentation20/architecture/#数据模型)：关系型数据库模型，但要求每个采集点单独建表 -- [集群与基本逻辑单元](https://www.taosdata.com/cn/documentation20/architecture/#集群与基本逻辑单元)：吸取NoSQL优点，支持水平扩展，支持高可靠 -- [存储模型与数据分区、分片](https://www.taosdata.com/cn/documentation20/architecture/#存储模型与数据分区、分片)：标签数据与时序数据完全分离，按vnode和时间两个维度对数据切分 -- [数据写入与复制流程](https://www.taosdata.com/cn/documentation20/architecture/#数据写入与复制流程)：先写入WAL、之后写入缓存，再给应用确认，支持多副本 -- [缓存与持久化](https://www.taosdata.com/cn/documentation20/architecture/#缓存与持久化)：最新数据缓存在内存中，但落盘时采用列式存储、超高压缩比 -- [数据查询](https://www.taosdata.com/cn/documentation20/architecture/#数据查询)：支持各种函数、时间轴聚合、插值、多表聚合 - -## [数据建模](https://www.taosdata.com/cn/documentation20/model) - -- [创建库](https://www.taosdata.com/cn/documentation20/model/#创建库)：为具有相似数据特征的数据采集点创建一个库 -- [创建超级表](https://www.taosdata.com/cn/documentation20/model/#创建超级表)：为同一类型的数据采集点创建一个超级表 -- [创建表](https://www.taosdata.com/cn/documentation20/model/#创建表)：使用超级表做模板，为每一个具体的数据采集点单独建表 - -## [高效写入数据](https://www.taosdata.com/cn/documentation20/insert) - -- [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入)：使用SQL insert命令向一张或多张表写入单条或多条记录 -- [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入)：配置Telegraf, 不用任何代码，将采集数据直接写入 -- [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入)：配置Prometheus, 不用任何代码，将数据直接写入 -- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将 MQTT 数据直接写入 -- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入)：通过 HiveMQ Extension，不用任何代码，就可将 MQTT 数据直接写入 - -## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries) - -- [主要查询功能](https://www.taosdata.com/cn/documentation20/queries/#主要查询功能)：支持各种标准函数，设置过滤条件，时间段查询 -- [多表聚合查询](https://www.taosdata.com/cn/documentation20/queries/#多表聚合查询)：使用超级表，设置标签过滤条件，进行高效聚合查询 -- [降采样查询值](https://www.taosdata.com/cn/documentation20/queries/#降采样查询、插值)：按时间段分段聚合，支持插值 - -## [高级功能](https://www.taosdata.com/cn/documentation20/advanced-features) - -- [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation20/advanced-features/#连续查询(Continuous-Query))：基于滑动窗口，定时自动的对数据流进行查询计算 -- [数据订阅(Publisher/Subscriber)](https://www.taosdata.com/cn/documentation20/advanced-features/#数据订阅(Publisher/Subscriber))：象典型的消息队列，应用可订阅接收到的最新数据 -- [缓存(Cache)](https://www.taosdata.com/cn/documentation20/advanced-features/#缓存(Cache))：每个设备最新的数据都会缓存在内存中，可快速获取 -- [报警监测](https://www.taosdata.com/cn/documentation20/advanced-features/#报警监测(Alert))：根据配置规则，自动监测超限行为数据，并主动推送 - -## [连接器](https://www.taosdata.com/cn/documentation20/connector) - -- [C/C++ Connector](https://www.taosdata.com/cn/documentation20/connector/#C/C++-Connector)：通过libtaos客户端的库，连接TDengine服务器的主要方法 -- [Java Connector(JDBC)](https://www.taosdata.com/cn/documentation20/connector-java)：通过标准的JDBC API，给Java应用提供到TDengine的连接 -- [Python Connector](https://www.taosdata.com/cn/documentation20/connector/#Python-Connector)：给Python应用提供一个连接TDengine服务器的驱动 -- [RESTful Connector](https://www.taosdata.com/cn/documentation20/connector/#RESTful-Connector)：提供一最简单的连接TDengine服务器的方式 -- [Go Connector](https://www.taosdata.com/cn/documentation20/connector/#Go-Connector)：给Go应用提供一个连接TDengine服务器的驱动 -- [Node.js Connector](https://www.taosdata.com/cn/documentation20/connector/#Node.js-Connector)：给node应用提供一个链接TDengine服务器的驱动 - -## [与其他工具的连接](https://www.taosdata.com/cn/documentation20/connections-with-other-tools) - -- [Grafana](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#Grafana)：获取并可视化保存在TDengine的数据 -- [Matlab](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#Matlab)：通过配置Matlab的JDBC数据源访问保存在TDengine的数据 -- [R](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#R)：通过配置R的JDBC数据源访问保存在TDengine的数据 - -## [TDengine集群的安装、管理](https://www.taosdata.com/cn/documentation20/cluster) - -- [安装](https://www.taosdata.com/cn/documentation20/cluster/#创建第一个节点)：与单节点的安装一样，但要设好配置文件里的参数first -- [节点管理](https://www.taosdata.com/cn/documentation20/cluster/#节点管理)：增加、删除、查看集群的节点 -- [mnode的管理](https://www.taosdata.com/cn/documentation20/cluster/#Mnode的高可用)：系统自动创建、无需任何人工干预 -- [负载均衡](https://www.taosdata.com/cn/documentation20/cluster/#负载均衡)：一旦节点个数或负载有变化，自动进行 -- [节点离线处理](https://www.taosdata.com/cn/documentation20/cluster/#节点离线处理)：节点离线超过一定时长，将从集群中剔除 -- [Arbitrator](https://www.taosdata.com/cn/documentation20/cluster/#Arbitrator的使用)：对于偶数个副本的情形，使用它可以防止split brain - -## [TDengine的运营和维护](https://www.taosdata.com/cn/documentation20/administrator) - -- [容量规划](https://www.taosdata.com/cn/documentation20/administrator/#容量规划)：根据场景，估算硬件资源 -- [容错和灾备](https://www.taosdata.com/cn/documentation20/administrator/#容错和灾备)：设置正确的WAL和数据副本数 -- [系统配置](https://www.taosdata.com/cn/documentation20/administrator/#服务端配置)：端口，缓存大小，文件块大小和其他系统配置 -- [用户管理](https://www.taosdata.com/cn/documentation20/administrator/#用户管理)：添加、删除TDengine用户，修改用户密码 -- [数据导入](https://www.taosdata.com/cn/documentation20/administrator/#数据导入)：可按脚本文件导入，也可按数据文件导入 -- [数据导出](https://www.taosdata.com/cn/documentation20/administrator/#数据导出)：从shell按表导出，也可用taosdump工具做各种导出 -- [系统监控](https://www.taosdata.com/cn/documentation20/administrator/#系统监控)：检查系统现有的连接、查询、流式计算，日志和事件等 -- [文件目录结构](https://www.taosdata.com/cn/documentation20/administrator/#文件目录结构)：TDengine数据文件、配置文件等所在目录 - -## [TAOS SQL](https://www.taosdata.com/cn/documentation20/taos-sql) - -- [支持的数据类型](https://www.taosdata.com/cn/documentation20/taos-sql/#支持的数据类型)：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 -- [数据库管理](https://www.taosdata.com/cn/documentation20/taos-sql/#数据库管理)：添加、删除、查看数据库 -- [表管理](https://www.taosdata.com/cn/documentation20/taos-sql/#表管理)：添加、删除、查看、修改表 -- [超级表管理](https://www.taosdata.com/cn/documentation20/taos-sql/#超级表STable管理)：添加、删除、查看、修改超级表 -- [标签管理](https://www.taosdata.com/cn/documentation20/taos-sql/#超级表-STable-中-TAG-管理)：增加、删除、修改标签 -- [数据写入](https://www.taosdata.com/cn/documentation20/taos-sql/#数据写入)：支持单表单条、多条、多表多条写入，支持历史数据写入 -- [数据查询](https://www.taosdata.com/cn/documentation20/taos-sql/#数据查询)：支持时间段、值过滤、排序、查询结果手动分页等 -- [SQL函数](https://www.taosdata.com/cn/documentation20/taos-sql/#SQL函数)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等 -- [时间维度聚合](https://www.taosdata.com/cn/documentation20/taos-sql/#时间维度聚合)：将表中数据按照时间段进行切割后聚合，降维处理 -- [边界线制](https://www.taosdata.com/cn/documentation20/taos-sql/#TAOS-SQL-边界限制)：TAOS SQL的边界限制 -- [错误码](https://www.taosdata.com/cn/documentation20/Taos-Error-Code)：TDengine 2.0 错误码以及对应的十进制码 - -## TDengine的技术设计 - -- 系统模块：taosd的功能和模块划分 -- 数据复制：支持实时同步、异步复制，保证系统的High Availibility -- [技术博客](https://www.taosdata.com/cn/blog/?categories=3)：更多的技术分析和架构设计文章 - -## 常用工具 - -- [TDengine样例导入工具](https://www.taosdata.com/blog/2020/01/18/1166.html) -- [TDengine性能对比测试工具](https://www.taosdata.com/blog/2020/01/18/1166.html) - -## TDengine与其他数据库的对比测试 - -- [用InfluxDB开源的性能测试工具对比InfluxDB和TDengine](https://www.taosdata.com/blog/2020/01/13/1105.html) -- [TDengine与OpenTSDB对比测试](https://www.taosdata.com/blog/2019/08/21/621.html) -- [TDengine与Cassandra对比测试](https://www.taosdata.com/blog/2019/08/14/573.html) -- [TDengine与InfluxDB对比测试](https://www.taosdata.com/blog/2019/07/19/419.html) -- [TDengine与InfluxDB、OpenTSDB、Cassandra、MySQL、ClickHouse等数据库的对比测试报告](https://www.taosdata.com/downloads/TDengine_Testing_Report_cn.pdf) - -##物联网大数据 - -- [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/105.html) -- [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/542.html) -- [通用大数据架构为什么不适合处理物联网数据？](https://www.taosdata.com/blog/2019/07/09/107.html) -- [物联网、车联网、工业互联网大数据平台，为什么推荐使用TDengine？](https://www.taosdata.com/blog/2019/07/09/109.html) - -## [培训和FAQ](https://www.taosdata.com/cn/faq) - -- [FAQ](https://www.taosdata.com/cn/documentation20/faq)：常见问题与答案 -- [应用案列](https://www.taosdata.com/cn/blog/?categories=4)：一些使用实例来解释如何使用TDengine \ No newline at end of file diff --git a/documentation20/webdocs/markdowndocs/Getting Started-ch.md b/documentation20/webdocs/markdowndocs/Getting Started-ch.md deleted file mode 100644 index beb0c639ae7f5d6b5535b194f70d806305208492..0000000000000000000000000000000000000000 --- a/documentation20/webdocs/markdowndocs/Getting Started-ch.md +++ /dev/null @@ -1,172 +0,0 @@ -# 立即开始 - -## 快捷安装 - -TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版仅能在Linux系统上安装和运行，后续会支持Windows、MAC OS等系统。如果应用需要在Windows或Mac上运行，目前只能使用TDengine的RESTful接口连接服务器。硬件支持X64，后续会支持ARM、龙芯等CPU系统。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 - -### 通过源码安装 - -请参考我们的[TDengine github主页](https://github.com/taosdata/TDengine)下载源码并安装. - -### 通过Docker容器运行 - -请参考[TDengine官方Docker镜像的发布、下载和使用](https://www.taosdata.com/blog/2020/05/13/1509.html) - -### 通过安装包安装 - -服务器部分，我们提供三种安装包，您可以根据需要选择。TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。 - - - -客户端部分，Linux安装包如下： - -- TDengine-client-2.0.0.0-Linux-x64.tar.gz (3.4M) - -报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）： - -- TDengine-alert-2.0.0-Linux-x64.tar.gz (8.1M) - -目前，TDengine只支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装。其他linux系统的支持正在开发中。用`which systemctl`命令来检测系统中是否存在`systemd`包: - -```cmd -which systemctl -``` - -如果系统中不存在`systemd`包，请考虑[通过源码安装](#通过源码安装)TDengine。 - -具体的安装过程，请参见TDengine多种安装包的安装和卸载。 - -## 轻松启动 - -安装成功后，用户可使用`systemctl`命令来启动TDengine的服务进程。 - -```cmd -systemctl start taosd -``` - -检查服务是否正常工作。 -```cmd -systemctl status taosd -``` - -如果TDengine服务正常工作，那么您可以通过TDengine的命令行程序`taos`来访问并体验TDengine。 - -**注意：** - -- systemctl命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo -- 为更好的获得产品反馈，改善产品，TDengine会采集基本的使用信息，但您可以修改系统配置文件taos.cfg里的配置参数telemetryReporting, 将其设为0，就可将其关闭。 - -## TDengine命令行程序 - -执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。 - -```cmd -taos -``` - -如果TDengine终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/faq/)来解决终端连接服务端失败的问题）。TDengine终端的提示符号如下： - -```cmd -taos> -``` - -在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例： - -```mysql -create database demo; -use demo; -create table t (ts timestamp, speed int); -insert into t values ('2019-07-15 00:00:00', 10); -insert into t values ('2019-07-15 01:00:00', 20); -select * from t; - ts | speed | -=================================== - 19-07-15 00:00:00.000| 10| - 19-07-15 01:00:00.000| 20| -Query OK, 2 row(s) in set (0.001700s) -``` - -除执行SQL语句外，系统管理员还可以从TDengine终端检查系统运行状态，添加删除用户账号等。 - -### 命令行参数 - -您可通过配置命令行参数来改变TDengine终端的行为。以下为常用的几个命令行参数： - -- -c, --config-dir: 指定配置文件目录，默认为_/etc/taos_ -- -h, --host: 指定服务的IP地址，默认为本地服务 -- -s, --commands: 在不进入终端的情况下运行TDengine命令 -- -u, -- user: 连接TDengine服务器的用户名，缺省为root -- -p, --password: 连接TDengine服务器的密码，缺省为taosdata -- -?, --help: 打印出所有命令行参数 - -示例： - -```cmd -taos -h 192.168.0.1 -s "use db; show tables;" -``` - -### 运行SQL命令脚本 - -TDengine终端可以通过`source`命令来运行SQL命令脚本. - -``` -taos> source ; -``` - -### Shell小技巧 - -- 可以使用上下光标键查看已经历史输入的命令 -- 修改用户密码。在shell中使用alter user命令 -- ctrl+c 中止正在进行中的查询 -- 执行`RESET QUERY CACHE`清空本地缓存的表的schema - -## TDengine 极速体验 - -启动TDengine的服务，在Linux终端执行taosdemo - -``` -> taosdemo -``` - -该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。 - -执行这条命令大概需要10分钟，最后共插入10亿条记录。 - -在TDengine客户端输入查询命令，体验查询速度。 - -- 查询超级表下记录总条数： - -``` -taos>select count(*) from test.meters; -``` - -- 查询10亿条记录的平均值、最大值、最小值等： - -``` -taos>select avg(f1), max(f2), min(f3) from test.meters; -``` - -- 查询loc="beijing"的记录总条数： - -``` -taos>select count(*) from test.meters where loc="beijing"; -``` - -- 查询areaid=10的所有记录的平均值、最大值、最小值等： - -``` -taos>select avg(f1), max(f2), min(f3) from test.meters where areaid=10; -``` - -- 对表t10按10s进行平均值、最大值和最小值聚合统计： - -``` -taos>select avg(f1), max(f2), min(f3) from test.t10 interval(10s); -``` - -**Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。 - diff --git a/documentation20/webdocs/markdowndocs/Super Table-ch.md b/documentation20/webdocs/markdowndocs/Super Table-ch.md index 96e7104ab7fe78a0015b596dd31bec6a849f2f6a..e5c77471570a76e608d59a0dca10462315460337 100644 --- a/documentation20/webdocs/markdowndocs/Super Table-ch.md +++ b/documentation20/webdocs/markdowndocs/Super Table-ch.md @@ -161,7 +161,7 @@ SELECT function,… 超级表聚合查询，TDengine目前支持以下聚合\选择函数：sum、count、avg、first、last、min、max、top、bottom，以及针对全部或部分列的投影操作，使用方式与单表查询的计算过程相同。暂不支持其他类型的聚合计算和四则运算。当前所有的函数及计算过程均不支持嵌套的方式进行执行。 - 不使用GROUP BY的查询将会对超级表下所有满足筛选条件的表按时间进行聚合，结果输出默认是按照时间戳单调递增输出，用户可以使用ORDER BY _c0 ASC|DESC选择查询结果时间戳的升降排序；使用GROUP BY 的聚合查询会按照tags进行分组，并对每个组内的数据分别进行聚合，输出结果为各个组的聚合结果，组间的排序可以由ORDER BY 语句指定，每个分组内部，时间序列是单调递增的。 + 不使用GROUP BY的查询将会对超级表下所有满足筛选条件的表按时间进行聚合，结果输出默认是按照时间戳单调递增输出，用户可以使用ORDER BY _c0 ASC|DESC选择查询结果时间戳的升降排序；使用GROUP BY 的聚合查询会按照tags进行分组，并对每个组内的数据分别进行聚合，输出结果为各个组的聚合结果，组间的排序可以由ORDER BY 语句指定，每个分组内部，时间序列是单调递增的。 使用SLIMIT/SOFFSET语句指定组间分页，即指定结果集中输出的最大组数以及对组起始的位置。使用LIMIT/OFFSET语句指定组内分页，即指定结果集中每个组内最多输出多少条记录以及记录起始的位置。 @@ -178,7 +178,7 @@ CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int) ``` -假设有北京，天津和上海三个地区的采集器共4个，温度采集器有3种类型，我们就可以对每个采集器建表如下： +假设有北京，天津和上海三个地区的采集器共4个，温度采集器有3种类型，我们就可以对每个采集器建表如下： ```mysql CREATE TABLE therm1 USING thermometer TAGS (’beijing’, 1); diff --git a/documentation20/webdocs/markdowndocs/cluster.md b/documentation20/webdocs/markdowndocs/cluster.md deleted file mode 100644 index f5fa6af48c9bb0a4062c9de74ee3555561ac26ba..0000000000000000000000000000000000000000 --- a/documentation20/webdocs/markdowndocs/cluster.md +++ /dev/null @@ -1,142 +0,0 @@ -#集群安装、管理 - -多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。 - -集群的每个节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令“hostname"获取。端口是这个节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。 - -TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 - -##安装、创建第一个节点 - -集群是由一个一个dnode组成的，是从一个dnode的创建开始的。创建第一个节点很简单，就按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装、启动即可。 - -启动后，请执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： - ``` -Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 -Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | -===================================================================================== - 1 | h1.taos.com:6030 | 0 | 2 | ready | any | 2020-07-31 03:49:29.202 | -Query OK, 1 row(s) in set (0.006385s) - -taos> - ``` -上述命令里，可以看到这个刚启动的这个节点的End Point是：h1.taos.com:6030 - -## 安装、创建后续节点 - -将新的节点添加到现有集群，具体有以下几步： - -1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装，但不要启动taosd - -2. 如果是使用涛思数据的官方安装包进行安装，在安装结束时，会询问集群的End Port, 输入第一个节点的End Point即可。如果是源码安装，请编辑配置文件taos.cfg(缺省是在/etc/taos/目录)，增加一行： - - ``` - firstEp h1.taos.com:6030 - ``` - - 请注意将示例的“h1.taos.com:6030" 替换为你自己第一个节点的End Point - -3. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法启动taosd - -4. 在Linux shell里执行命令"hostname"找出本机的FQDN, 假设为h2.taos.com。如果无法找到，可以查看taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)，fqdn以及port都会打印出来。 - -5. 在第一个节点，使用CLI程序taos, 登录进TDengine系统, 使用命令: - - ``` - CREATE DNODE "h2.taos.com:6030"; - ``` - - 将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为你自己第一个节点的End Point - -6. 使用命令 - - ``` - SHOW DNODES; - ``` - - 查看新节点是否被成功加入。 - -按照上述步骤可以源源不断的将新的节点加入到集群。 - -**提示：** - -- firstEp, secondEp这两个参数仅仅在该节点第一次加入集群时有作用，加入集群后，该节点会保存最新的mnode的End Point列表，不再依赖这两个参数。 -- 两个没有配置first, second参数的dnode启动后，会独立运行起来。这个时候，无法将其中一个节点加入到另外一个节点，形成集群。**无法将两个独立的集群合并成为新的集群**。 - -##节点管理 - -###添加节点 -执行CLI程序taos, 使用root账号登录进系统, 执行: -``` -CREATE DNODE "fqdn:port"; -``` -将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。 - -###删除节点 -执行CLI程序taos, 使用root账号登录进TDengine系统，执行： -``` -DROP DNODE "fqdn:port"; -``` -其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号 - -###查看节点 -执行CLI程序taos,使用root账号登录进TDengine系统，执行： -``` -SHOW DNODES; -``` -它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等），vnode数目，还未使用的vnode数目等信息。在添加或删除一个节点后，可以使用该命令查看。 - -###查看虚拟节点组 - -为充分利用多核技术，并提供scalability，数据需要分片处理。因此TDengine会将一个DB的数据切分成多份，存放在多个vnode里。这些vnode可能分布在多个dnode里，这样就实现了水平扩展。一个vnode仅仅属于一个DB，但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 - -执行CLI程序taos,使用root账号登录进TDengine系统，执行： -``` -SHOW VGROUPS; -``` -##高可用性 -TDengine通过多副本的机制来提供系统的高可用性。副本数是与DB关联的，一个集群里可以有多个DB，根据运营的需求，每个DB可以配置不同的副本数。创建数据库时，通过参数replica 指定副本数（缺省为1）。如果副本数为1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed"。比如下面的命令将创建副本数为3的数据库demo： -``` -CREATE DATABASE demo replica 3; -``` -一个DB里的数据会被切片分到多个vnode group，vnode group里的vnode数目就是DB的副本数，同一个vnode group里各vnode的数据是完全一致的。为保证高可用性，vnode group里的vnode一定要分布在不同的dnode里（实际部署时，需要在不同的物理机上），只要一个vgroup里超过半数的vnode处于工作状态，这个vgroup就能正常的对外服务。 - -一个dnode里可能有多个DB的数据，因此一个dnode离线时，可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作，那么该vnode group就无法对外服务，无法插入或读取数据，这样会影响到它所属的DB的一部分表的d读写操作。 - -因为vnode的引入，无法简单的给出结论：“集群中过半dnode工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为3，只有三个dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个节点不工作，那整个集群就无法正常工作了。 - -##Mnode的高可用 -TDengine集群是由mnode (taosd的一个模块，逻辑节点) 负责管理的，为保证mnode的高可用，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，有效范围为1-3。为保证元数据的强一致性，mnode副本之间是通过同步的方式进行数据复制的。 - -一个集群有多个dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下，哪个dnode可以作为mnode呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过CLI程序taos，在TDengine的console里，执行如下命令： -``` -SHOW MNODES; -``` -来查看mnode列表，该列表将列出mnode所处的dnode的End Point和角色(master, slave, unsynced 或offline)。 -当集群中第一个节点启动时，该节点一定会运行一个mnode实例，否则该dnode无法正常工作，因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2，启动第二个dnode时，该dnode也将运行一个mnode实例。 - -为保证mnode服务的高可用性，numOfMnodes必须设置为2或更大。因为mnode保存的元数据必须是强一致的，如果numOfMnodes大于2，复制参数quorum自动设为2，也就是说，至少要保证有两个副本写入数据成功，才通知客户端应用写入成功。 - -##负载均衡 - -有三种情况，将触发负载均衡，而且都无需人工干预。 - -- 当一个新节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新节点上，无需任何人工干预。 -- 当一个节点从集群中移除时，系统将自动把该节点上的数据转移到其他节点，无需任何人工干预。 -- 如果一个节点过热（数据量过大），系统将自动进行负载均衡，将该节点的一些vnode自动挪到其他节点。 - -当上述三种情况发生时，系统将启动一各个节点的负载计算，从而决定如何挪动。 - -##节点离线处理 -如果一个节点离线，TDengine集群将自动检测到。有如下两种情况： -- 改节点离线超过一定时间（taos.cfg里配置参数offlineThreshold控制时长)，系统将自动把该节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的节点重现上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 -- 离线后，在offlineThreshold的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。 - -##Arbitrator的使用 - -如果副本数为偶数，当一个vnode group里一半vnode不工作时，是无法从中选出master的。同理，一半mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 - -TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6030。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 8a68d02dfd232e557d0877e738aa8aa33a02467f..a1178e2eefd6c4e6a4433a451d2e4e865a39cfe6 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -29,16 +29,23 @@ # number of threads per CPU core # numOfThreadsPerCore 1.0 -# the proportion of total threads responsible for query -# ratioOfQueryThreads 0.5 +# the proportion of total CPU cores available for query processing +# 2.0: the query threads will be set to double of the CPU cores. +# 1.0: all CPU cores are available for query processing [default]. +# 0.5: only half of the CPU cores are available for query. +# 0.0: only one core available. +# ratioOfQueryCores 1.0 + +# the last_row/first/last aggregator will not change the original column name in the result fields +# keepColumnName 0 # number of management nodes in the system # numOfMnodes 3 -# enable/disable backuping vnode directory when removing dnode +# enable/disable backuping vnode directory when removing vnode # vnodeBak 1 -# if report installation / use information +# enable/disable installation / usage report # telemetryReporting 1 # enable/disable load balancing @@ -68,16 +75,13 @@ # time interval of heart beat from shell to dnode, seconds # shellActivityTimer 3 -# time of keeping table meta data in cache, seconds -# tableMetaKeepTimer 7200 - # minimum sliding window time, milli-second # minSlidingTime 10 # minimum time window, milli-second # minIntervalTime 10 -# maximum delay before launching a stream compution, milli-second +# maximum delay before launching a stream computation, milli-second # maxStreamCompDelay 20000 # maximum delay before launching a stream computation for the first time, milli-second @@ -152,16 +156,16 @@ # max number of connections allowed in dnode # maxShellConns 5000 -# max numerber of connections allowed in client +# max number of connections allowed in client # maxConnections 5000 # stop writing logs when the disk size of the log folder is less than this value # minimalLogDirGB 0.1 -# stop writing temporary files when the disk size of the log folder is less than this value +# stop writing temporary files when the disk size of the tmp folder is less than this value # minimalTmpDirGB 0.1 -# stop writing data when the disk size of the log folder is less than this value +# if disk free space is less than this value, taosd service exit directly within startup process # minimalDataDirGB 0.1 # One mnode is equal to the number of vnode consumed @@ -183,7 +187,7 @@ # restfulRowLimit 10240 # The following parameter is used to limit the maximum number of lines in log files. -# max number of rows per log filters +# max number of lines per log filters # numOfLogLines 10000000 # enable/disable async log @@ -195,7 +199,9 @@ # The following parameters are used for debug purpose only. # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log. +# 131: output warning and error +# 135: output debug, warning and error +# 143: output trace, debug, warning and error to log # 199: output debug, warning and error to both screen and file # 207: output trace, debug, warning and error to both screen and file @@ -227,10 +233,10 @@ # cDebugFlag 131 # debug flag for JNI -# jniDebugflag 131 +# jniDebugFlag 131 # debug flag for storage -# uDebugflag 131 +# uDebugFlag 131 # debug flag for http server # httpDebugFlag 131 @@ -239,12 +245,12 @@ # monDebugFlag 131 # debug flag for query -# qDebugflag 131 +# qDebugFlag 131 # debug flag for vnode -# vDebugflag 131 +# vDebugFlag 131 -# debug flag for http server +# debug flag for TSDB # tsdbDebugFlag 131 # debug flag for continue query @@ -259,11 +265,14 @@ # maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden # maxBinaryDisplayWidth 30 -# enable/disable telemetry reporting -# telemetryReporting 1 - # enable/disable stream (continuous query) # stream 1 -# only 50% CPU resources will be used in query processing -# halfCoresForQuery 0 +# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode +# retrieveBlockingModel 0 + +# the maximum allowed query buffer size in MB during query processing for each data node +# -1 no limit (default) +# 0 no query allowed, queries are disabled +# queryBufferSize -1 + diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index edc7de96923266729d1985cb43abfa3003f9178e..36870b2ebe49d45390d7b5ce18f6984b9e8e2ac2 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -47,6 +47,9 @@ cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script +cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 44ccafd04460c8beef0a4b2f5ae715033328f807..230741d036a3012db1ea6351a2f30ca4b704d350 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -1,19 +1,20 @@ -FROM ubuntu:20.04 +FROM ubuntu:18.04 WORKDIR /root -ARG version -RUN echo $version -COPY tdengine.tar.gz /root/ -RUN tar -zxf tdengine.tar.gz -WORKDIR /root/TDengine-server-$version/ -RUN /bin/bash install.sh -e no +ARG pkgFile +ARG dirName +RUN echo ${pkgFile} +RUN echo ${dirName} +COPY ${pkgFile} /root/ +RUN tar -zxf ${pkgFile} +WORKDIR /root/${dirName}/ +RUN /bin/bash install.sh -e no ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 CMD ["taosd"] VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh new file mode 100755 index 0000000000000000000000000000000000000000..b52580cfa6bdc13fcc7b716a85029fd1c679a6b6 --- /dev/null +++ b/packaging/docker/dockerManifest.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e +#set -x + +# dockerbuild.sh +# -n [version number] +# -p [xxxx] + +# set parameters by default value +verNumber="" +passWord="" + +while getopts "hn:p:" arg +do + case $arg in + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "verNumber=${verNumber}" + +#docker manifest create -a tdengine/tdengine:${verNumber} tdengine/tdengine-amd64:${verNumber} tdengine/tdengine-aarch64:${verNumber} tdengine/tdengine-aarch32:${verNumber} +docker manifest create -a tdengine/tdengine tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + +docker login -u tdengine -p ${passWord} #replace the docker registry username and password + +docker manifest push tdengine/tdengine + +# how set latest version ??? diff --git a/packaging/docker/dockerbuild-aarch64.sh b/packaging/docker/dockerbuild-aarch64.sh deleted file mode 100755 index 795eaed5491834cf5601e567440a314d6e1f9629..0000000000000000000000000000000000000000 --- a/packaging/docker/dockerbuild-aarch64.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -x -docker build --rm -f "Dockerfile" -t tdengine/tdengine-aarch64:$1 "." --build-arg version=$1 -docker login -u tdengine -p $2 #replace the docker registry username and password -docker push tdengine/tdengine-aarch64:$1 diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 48e2f7ead63356e8210d03beff2add6796558f5b..b7991465b0940c86ac6d620d498901ab5e1c9ac2 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -1,5 +1,63 @@ #!/bin/bash -set -x -docker build --rm -f "Dockerfile" -t tdengine/tdengine:$1 "." --build-arg version=$1 -docker login -u tdengine -p $2 #replace the docker registry username and password -docker push tdengine/tdengine:$1 +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -f [pkg file] +# -n [version number] +# -p [password for docker hub] + +# set parameters by default value +cpuType=amd64 +verNumber="" +passWord="" +pkgFile="" + +while getopts "hc:n:p:f:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + f) + #echo "pkgFile=$OPTARG" + pkgFile=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -f [pkg file] " + echo " -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "cpuType=${cpuType} verNumber=${verNumber} pkgFile=${pkgFile} " +echo "$(pwd)" +echo "====NOTES: ${pkgFile} must be in the same directory as dockerbuild.sh====" + +dirName=${pkgFile%-Linux*} +#echo "dirName=${dirName}" + +docker build --rm -f "Dockerfile" -t tdengine/tdengine-${cpuType}:${verNumber} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} +docker login -u tdengine -p ${passWord} #replace the docker registry username and password +docker push tdengine/tdengine-${cpuType}:${verNumber} + +# set this version to latest version +docker tag tdengine/tdengine-${cpuType}:${verNumber} tdengine/tdengine-${cpuType}:latest +docker push tdengine/tdengine-${cpuType}:latest diff --git a/packaging/docker/dockerbuildi.sh b/packaging/docker/dockerbuildi.sh new file mode 100755 index 0000000000000000000000000000000000000000..a0a954e30fe9c3637abe4d219001235d793466e0 --- /dev/null +++ b/packaging/docker/dockerbuildi.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# + +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -n [version number] +# -p [password for docker hub] + +# set parameters by default value +cpuType=aarch64 +verNumber="" +passWord="" + +while getopts "hc:n:p:f:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +pkgFile=TDengine-server-${verNumber}-Linux-${cpuType}.tar.gz + +echo "cpuType=${cpuType} verNumber=${verNumber} pkgFile=${pkgFile} " + +scriptDir=`pwd` +pkgDir=$scriptDir/../../release/ + +cp -f ${pkgDir}/${pkgFile} . + +./dockerbuild.sh -c ${cpuType} -f ${pkgFile} -n ${verNumber} -p ${passWord} + +rm -f ${pkgFile} diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 8c23ab802d54088dafc4b4c6a5d1b6241c881f2e..92c917cb3d6d4cd5f41441c9ca75a742aa3641b6 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -9,6 +9,7 @@ Summary: tdengine from taosdata Group: Application/Database License: AGPL URL: www.taosdata.com +AutoReqProv: no #BuildRoot: %_topdir/BUILDROOT BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root @@ -55,6 +56,9 @@ cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script +cp %{_compiledir}/../packaging/tools/startPre.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/set_core.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin @@ -135,7 +139,7 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - #${csudo} rm -f ${bin_link_dir}/taosdump || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index d6dccf7045a02a3019d95d73685a3ca27bc5205f..dca3dd2ff623672eb85b3de72bcc34e0ea5e3d8a 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -9,6 +9,9 @@ set -e verMode=edge pagMode=full +iplist="" +serverFqdn="" + # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory @@ -144,8 +147,8 @@ done #echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" -function kill_taosd() { - pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo} kill -9 $pid || : fi @@ -165,6 +168,10 @@ function install_main_path() { if [ "$verMode" == "cluster" ]; then ${csudo} mkdir -p ${nginx_dir} fi + + if [[ -e ${script_dir}/email ]]; then + ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: + fi } function install_bin() { @@ -227,6 +234,157 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -247,7 +405,9 @@ function install_config() { if [ "$interactiveFqdn" == "no" ]; then return 0 - fi + fi + + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" @@ -446,6 +606,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" @@ -454,6 +615,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}" ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" + #${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}" ${csudo} bash -c "echo >> ${taosd_service_config}" ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" @@ -519,7 +681,7 @@ function install_service() { install_service_on_sysvinit else # must manual stop taosd - kill_taosd + kill_process taosd fi } @@ -588,9 +750,22 @@ function update_TDengine() { elif ((${service_mod}==1)); then ${csudo} service taosd stop || : else - kill_taosd + kill_process taosd + fi + sleep 1 + fi + + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx fi sleep 1 + fi fi install_main_path @@ -634,9 +809,9 @@ function update_TDengine() { fi if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" else - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" + echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell${NC}" fi echo @@ -655,8 +830,8 @@ function update_TDengine() { function install_TDengine() { # Start to install if [ ! -e taos.tar.gz ]; then - echo "File taos.tar.gz does not exist" - exit 1 + echo "File taos.tar.gz does not exist" + exit 1 fi tar -zxf taos.tar.gz @@ -702,41 +877,54 @@ function install_TDengine() { echo echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}" + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}" elif ((${service_mod}==1)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" else - echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" + echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" fi - if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" - else - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" - fi + #if [ ${openresty_work} = 'true' ]; then + # echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + #else + # echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" + #fi if [ ! -z "$firstEp" ]; then - echo - echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp${GREEN_DARK} to login into cluster, then${NC}" - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" + echo fi + echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo else # Only install client install_bin install_config - echo echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" fi touch ~/.taos_history - rm -rf $(tar -tf taos.tar.gz) } ## ==============================Main program starts from here============================ +serverFqdn=$(hostname) if [ "$verType" == "server" ]; then # Install server and client if [ -x ${bin_dir}/taosd ]; then diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 0467300953ca34d1f9b24ba9999ccd156a0d8d09..0a0a6633e376d084532abb5f490917abd1a173f2 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -21,7 +21,7 @@ else cd ${script_dir} script_dir="$(pwd)" data_dir="/var/lib/taos" - log_dir="~/TDengineLog" + log_dir=~/TDengine/log fi log_link_dir="/usr/local/taos/log" diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 26977e12f4b7ee780948399605f28603817f1aa2..8d7463366ff46bcae2822ee3e76dbc9b588f2a89 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -85,8 +85,8 @@ function install_bin() { # Remove links ${csudo} rm -f ${bin_link_dir}/power || : if [ "$osType" != "Darwin" ]; then - ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdump || : + ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdump || : fi ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/set_core || : diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index 5929b52afca59f5e4364aa294fe7cefec5f6b828..ba6ace400935c10caadd9426c0701c16b4f86baa 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -9,6 +9,8 @@ set -e verMode=edge pagMode=full +iplist="" +serverFqdn="" # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory @@ -144,8 +146,8 @@ done #echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" -function kill_powerd() { - pid=$(ps -ef | grep "powerd" | grep -v "grep" | awk '{print $2}') +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo} kill -9 $pid || : fi @@ -172,7 +174,6 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerd || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdump || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -183,7 +184,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : [ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd || : [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : - [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || : [ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : @@ -227,6 +227,157 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -248,6 +399,8 @@ function install_config() { if [ "$interactiveFqdn" == "no" ]; then return 0 fi + + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" @@ -423,6 +576,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${powerd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${powerd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/powerd' >> ${powerd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/power/bin/startPre.sh' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${powerd_service_config}" @@ -496,7 +650,7 @@ function install_service() { install_service_on_sysvinit else # must manual stop powerd - kill_powerd + kill_process powerd fi } @@ -565,9 +719,21 @@ function update_PowerDB() { elif ((${service_mod}==1)); then ${csudo} service powerd stop || : else - kill_powerd + kill_process powerd + fi + sleep 1 + fi + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx fi sleep 1 + fi fi install_main_path @@ -611,9 +777,9 @@ function update_PowerDB() { fi if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" else - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" + echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell${NC}" fi echo @@ -686,17 +852,30 @@ function install_PowerDB() { echo -e "${GREEN_DARK}To start PowerDB ${NC}: powerd${NC}" fi - if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" - else - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" - fi + #if [ ${openresty_work} = 'true' ]; then + # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + #else + # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" + #fi if [ ! -z "$firstEp" ]; then - echo - echo -e "${GREEN_DARK}Please run${NC}: power -h $firstEp${GREEN_DARK} to login into cluster, then${NC}" + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $serverFqdn${GREEN_DARK} to login into PowerDB server${NC}" + echo fi echo -e "\033[44;32;1mPowerDB is installed successfully!${NC}" echo @@ -713,6 +892,7 @@ function install_PowerDB() { ## ==============================Main program starts from here============================ +serverFqdn=$(hostname) if [ "$verType" == "server" ]; then # Install server and client if [ -x ${bin_dir}/powerd ]; then diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 831012851ad70d05080bfae161c0b925d5215ae9..1fd0e943b155d0c96c26c3fdcec7061bd25197e4 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -24,7 +24,7 @@ data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then log_dir="/var/log/taos" else - log_dir="~/TDengineLog" + log_dir=~/TDengine/log fi data_link_dir="/usr/local/taos/data" @@ -149,10 +149,12 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin if [ "$osType" != "Darwin" ]; then - ${csudo} cp -r ${script_dir}/remove.sh ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/remove.sh ${install_main_dir}/bin ${csudo} cp -r ${script_dir}/set_core.sh ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/startPre.sh ${install_main_dir}/bin else ${csudo} cp -r ${script_dir}/remove_client.sh ${install_main_dir}/bin fi @@ -178,7 +180,9 @@ function install_bin() { function install_lib() { # Remove links ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + fi if [ "$osType" != "Darwin" ]; then ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${verNumber} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* @@ -190,12 +194,14 @@ function install_lib() { ${csudo} ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so fi else - ${csudo} cp ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - ${csudo} ldconfig + if [ "$osType" != "Darwin" ]; then + ${csudo} ldconfig + fi } function install_header() { @@ -326,6 +332,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index ee79a560407c650de8c511964f611cf8cacfd2d5..30e9fa51a7d9c4a98d2c8f300287ebd242fecd74 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,8 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \ + ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else @@ -54,7 +55,11 @@ else fi header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi install_files="${script_dir}/install_client.sh" diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index fdb3e0e5cc8c7add9c8ea8cdc9224964b3c80153..181536b7f19d252164201291d9c37cade6cf3490 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -54,7 +54,11 @@ else fi header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi install_files="${script_dir}/install_client_power.sh" @@ -80,6 +84,7 @@ if [ "$osType" != "Darwin" ]; then cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin fi else cp ${bin_files} ${install_dir}/bin diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 5ae5cbbcdc00c1fd862501a6eb0dccd3ac0fa007..36b1fe5bd88950f69a56e84d98fba9c4dae0cf05 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,12 +36,18 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator\ + ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" fi lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + install_files="${script_dir}/install.sh" nginx_dir="${code_dir}/../../enterprise/src/plugins/web" diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 13849484695f98a5fc206d88ec6a5de0cc930a9b..554e7884b1c3db69acd3ba0e3234e468b1d31c79 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -36,12 +36,17 @@ fi # strip ${build_dir}/bin/taos # bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${script_dir}/remove_power.sh" #else -# bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh ${script_dir}/set_core.sh" +# bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh\ +# ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" #fi lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi install_files="${script_dir}/install_power.sh" nginx_dir="${code_dir}/../../enterprise/src/plugins/web" @@ -81,6 +86,8 @@ else cp ${build_dir}/bin/tarbitrator ${install_dir}/bin cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/startPre.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin fi chmod a+x ${install_dir}/bin/* || : diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 52919976ee8a1b8c3ab7c38d9dad4eb7e98b22bd..8665b3fec3a392b3dcae8c6a197625ba85ed953b 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -3,6 +3,10 @@ # This file is used to install tdengine rpm package on centos systems. The operating system # is required to use systemd to manage services at boot #set -x + +iplist="" +serverFqdn="" + # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory @@ -106,12 +110,171 @@ function install_bin() { [ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : } +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + function install_config() { if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} ${csudo} mkdir -p ${cfg_install_dir} [ -f ${cfg_dir}/taos.cfg ] && ${csudo} cp ${cfg_dir}/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi + + # Save standard input to 6 and open / dev / TTY on standard input + exec 6<&0 0> ${taosd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" @@ -267,7 +431,7 @@ function install_service() { } function install_TDengine() { - echo -e "${GREEN}Start to install TDEngine...${NC}" + echo -e "${GREEN}Start to install TDengine...${NC}" #install log and data dir , then ln to /usr/local/taos ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} @@ -300,18 +464,32 @@ function install_TDengine() { echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}" fi - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" - + + if [ ! -z "$firstEp" ]; then - echo - echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp${GREEN_DARK} to login into cluster, then${NC}" - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo - fi + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" + echo + fi echo echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" } ## ==============================Main program starts from here============================ +serverFqdn=$(hostname) install_TDengine diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 07b43c0e49d56fd136747399c3c71fdb1885bea3..6c1d53606bc37c6a12787b672ccb0697ad0fe0b8 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -119,4 +119,4 @@ if ((${service_mod}==2)); then kill_taosd fi -echo -e "${GREEN}TDEngine is removed successfully!${NC}" +echo -e "${GREEN}TDengine is removed successfully!${NC}" diff --git a/packaging/tools/set_core.sh b/packaging/tools/set_core.sh index 083b0516dbb03daafe8c19548b7fac494af7d24a..f515fd3dc5ed4a26fc7f4120573d39303ea3d751 100755 --- a/packaging/tools/set_core.sh +++ b/packaging/tools/set_core.sh @@ -2,19 +2,39 @@ # # This file is used to set config for core when taosd crash -set -e +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +# set -e # set -x +corePath=$1 csudo="" if command -v sudo > /dev/null; then csudo="sudo" fi -#ulimit -c unlimited +if [[ ! -n ${corePath} ]]; then + echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:" + read corePath + while true; do + if [[ ! -z "$corePath" ]]; then + break + else + read -p "Please enter a file directory to save the coredump file:" corePath + fi + done +fi + +ulimit -c unlimited ${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||: ${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||: source /etc/profile -${csudo} mkdir -p /coredump ||: -${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p' ||: -${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern ||: +${csudo} mkdir -p ${corePath} ||: +${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: +${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern ||: diff --git a/packaging/tools/startPre.sh b/packaging/tools/startPre.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c16a5a938bdf3d7dc36c0a79f46e9c8f32b222b --- /dev/null +++ b/packaging/tools/startPre.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# if enable core dump, set start count to 3, disable core dump, set start count to 20. +# set -e +# set -x + +taosd=/etc/systemd/system/taosd.service +line=`grep StartLimitBurst ${taosd}` +num=${line##*=} +#echo "burst num: ${num}" + +startSeqFile=/usr/local/taos/.startSeq +recordFile=/usr/local/taos/.startRecord + +startSeq=0 + +if [[ ! -e ${startSeqFile} ]]; then + startSeq=0 +else + startSeq=$(cat ${startSeqFile}) +fi + +nextSeq=`expr $startSeq + 1` +echo "${nextSeq}" > ${startSeqFile} + +curTime=$(date "+%Y-%m-%d %H:%M:%S") +echo "startSeq:${startSeq} startPre.sh exec ${curTime}, burstCnt:${num}" >> ${recordFile} + + +coreFlag=`ulimit -c` +echo "coreFlag: ${coreFlag}" >> ${recordFile} + +if [ ${coreFlag} = "0" ];then + #echo "core is 0" + if [ ${num} != "20" ];then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=20/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 20" >> ${recordFile} + fi +fi + +if [ ${coreFlag} = "unlimited" ];then + #echo "core is unlimited" + if [ ${num} != "3" ];then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=3/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 3" >> ${recordFile} + fi +fi + diff --git a/packaging/tools/taosd-dump-cfg.gdb b/packaging/tools/taosd-dump-cfg.gdb new file mode 100644 index 0000000000000000000000000000000000000000..9774ccf82283817edea5f49b59e0c6cb6f529577 --- /dev/null +++ b/packaging/tools/taosd-dump-cfg.gdb @@ -0,0 +1,144 @@ +# Usage: +# sudo gdb -x ./taosd-dump-cfg.gdb + +define attach_pidof + if $argc != 1 + help attach_pidof + else + shell echo -e "\ +set \$PID = "$(echo $(pidof $arg0) 0 | cut -d " " -f 1)"\n\ +if \$PID > 0\n\ + attach "$(pidof -s $arg0)"\n\ +else\n\ + print \"Process '"$arg0"' not found\"\n\ +end" > /tmp/gdb.pidof + source /tmp/gdb.pidof + end +end + +document attach_pidof +Attach to process by name +Usage: attach_pidof PROG_NAME +end + +set $TAOS_CFG_VTYPE_INT8 = 0 +set $TAOS_CFG_VTYPE_INT16 = 1 +set $TAOS_CFG_VTYPE_INT32 = 2 +set $TAOS_CFG_VTYPE_FLOAT = 3 +set $TAOS_CFG_VTYPE_STRING = 4 +set $TAOS_CFG_VTYPE_IPSTR = 5 +set $TAOS_CFG_VTYPE_DIRECTORY = 6 + +set $TSDB_CFG_CTYPE_B_CONFIG = 1U +set $TSDB_CFG_CTYPE_B_SHOW = 2U +set $TSDB_CFG_CTYPE_B_LOG = 4U +set $TSDB_CFG_CTYPE_B_CLIENT = 8U +set $TSDB_CFG_CTYPE_B_OPTION = 16U +set $TSDB_CFG_CTYPE_B_NOT_PRINT = 32U + +set $TSDB_CFG_PRINT_LEN = 53 + +define print_blank + if $argc == 1 + set $blank_len = $arg0 + while $blank_len > 0 + printf "%s", " " + set $blank_len = $blank_len - 1 + end + end +end + +define dump_cfg + if $argc != 1 + help dump_cfg + else + set $blen = $TSDB_CFG_PRINT_LEN - (int)strlen($arg0.option) + if $blen < 0 + $blen = 0 + end + #printf "%s: %d\n", "******blen: ", $blen + printf "%s: ", $arg0.option + print_blank $blen + + if $arg0.valType == $TAOS_CFG_VTYPE_INT8 + printf "%d\n", *((int8_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT16 + printf "%d\n", *((int16_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT32 + printf "%d\n", *((int32_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_FLOAT + printf "%f\n", *((float *) $arg0.ptr) + else + printf "%s\n", $arg0.ptr + end + end + end + end + end +end + +document dump_cfg +Dump a cfg entry +Usage: dump_cfg cfg +end + +set pagination off + +attach_pidof taosd + +set $idx=0 +#print tsGlobalConfigNum +#set $end=$1 +set $end=tsGlobalConfigNum + +p "*=*=*=*=*=*=*=*=*= taos global config:" +#while ($idx .lt. $end) +while ($idx < $end) + # print tsGlobalConfig[$idx].option + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded +# p "1" + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +set $idx=0 + +p "*=*=*=*=*=*=*=*=*= taos local config:" +while ($idx < $end) + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if ($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +detach + +quit diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 059c0650c2bbba99694253a73b59d77cd16f06bf..7b6bfee42bd1a533798365edecacdff528420d73 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.8.0' +version: '2.0.16.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.8.0 + - usr/lib/libtaos.so.2.0.16.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a2600785c35a80d040b9f91137c630d7761cef7e..b0f2cc0a48f906b40d7be5185ae5f081c2ed4418 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,14 +5,13 @@ PROJECT(TDengine) ADD_SUBDIRECTORY(os) ADD_SUBDIRECTORY(common) ADD_SUBDIRECTORY(util) +ADD_SUBDIRECTORY(tfs) ADD_SUBDIRECTORY(rpc) ADD_SUBDIRECTORY(client) ADD_SUBDIRECTORY(query) ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) -IF (TD_SYNC) - ADD_SUBDIRECTORY(sync) -ENDIF () +ADD_SUBDIRECTORY(sync) ADD_SUBDIRECTORY(balance) ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(vnode) diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index 6c26e50b873d30b95c67a7988e83140106f609eb..967635e52ce20761dbd674a380563deeeb9af189 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -8,6 +8,4 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) - ADD_LIBRARY(balance ${SRC}) -ENDIF () +ADD_LIBRARY(balance ${SRC}) diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h index 8f54b66028c5ddc859861bb78e0d790f7d3e5507..74a761299da9bb0816869dae3fd49a6afb6350bd 100644 --- a/src/balance/inc/bnThread.h +++ b/src/balance/inc/bnThread.h @@ -24,7 +24,7 @@ extern "C" { int32_t bnInitThread(); void bnCleanupThread(); void bnNotify(); -void bnStartTimer(int64_t mseconds); +void bnStartTimer(int32_t mseconds); #ifdef __cplusplus } diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 383f98191313d3a40f0899787c0f421f2d741cef..236b22afafb9ed6ee13acfaf6b831520f2b4d2f6 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "tref.h" #include "tsync.h" #include "tglobal.h" #include "dnode.h" @@ -28,7 +29,9 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -static SBnMgmt tsBnMgmt;; +extern int64_t tsDnodeRid; +extern int32_t tsSdbRid; +static SBnMgmt tsBnMgmt; static void bnMonitorDnodeModule(); static void bnLock() { @@ -41,7 +44,7 @@ static void bnUnLock() { static bool bnCheckFree(SDnodeObj *pDnode) { if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status)); + mError("dnode:%d, status:%s not available", pDnode->dnodeId, dnodeStatus[pDnode->status]); return false; } @@ -63,6 +66,77 @@ static bool bnCheckFree(SDnodeObj *pDnode) { return true; } +static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { + SVnodeGid tmp = *pVnodeGid1; + *pVnodeGid1 = *pVnodeGid2; + *pVnodeGid2 = tmp; +} + +static void bnAdjustVnodeIndex(SVgObj *pInVg) { + int32_t d0Id = pInVg->vnodeGid[0].dnodeId; + int32_t d1Id = pInVg->vnodeGid[1].dnodeId; + int32_t d2Id = pInVg->vnodeGid[2].dnodeId; + + int32_t vgId = pInVg->vgId; + int32_t d0Num = 0; + int32_t d1Num = 0; + int32_t d2Num = 0; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + if (pVgroup->vgId != vgId) { + if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++; + if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++; + if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++; + } + + mnodeDecVgroupRef(pVgroup); + } + + if (pInVg->numOfVnodes == 1) { + } + + if (pInVg->numOfVnodes == 2) { + mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num); + if (d0Num > d1Num) { + mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); + } + } + + if (pInVg->numOfVnodes >= 3) { + mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num); + if (d0Num <= d1Num && d0Num <= d2Num) { + if (d1Num > d2Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } else if (d1Num <= d2Num && d1Num <= d0Num) { + mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); + if (d0Num > d2Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } else { + mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]); + if (d1Num > d0Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } + } + + for (int i = 0; i < pInVg->numOfVnodes; ++i) { + mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId); + } +} + static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId); @@ -85,17 +159,11 @@ static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); pVgroup->numOfVnodes = numOfVnodes; + bnAdjustVnodeIndex(pVgroup); mnodeUpdateVgroup(pVgroup); } -static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { - // SVnodeGid tmp = *pVnodeGid1; - // *pVnodeGid1 = *pVnodeGid2; - // *pVnodeGid2 = tmp; -} - int32_t bnAllocVnodes(SVgObj *pVgroup) { - static int32_t randIndex = 0; int32_t dnode = 0; int32_t vnodes = 0; @@ -117,8 +185,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { break; } else { mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, - mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable, - pDnode->alternativeRole); + dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); } } } @@ -134,7 +201,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { while (1) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; - mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), + mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); mnodeDecDnodeRef(pDnode); } @@ -146,36 +213,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { } } - /* - * make the choice more random. - * replica 1: no choice - * replica 2: there are 2 combinations - * replica 3 or larger: there are 6 combinations - */ - if (pVgroup->numOfVnodes == 1) { - } else if (pVgroup->numOfVnodes == 2) { - if (randIndex++ % 2 == 0) { - bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1); - } - } else { - int32_t randVal = randIndex++ % 6; - if (randVal == 1) { // 1, 0, 2 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); - } else if (randVal == 2) { // 1, 2, 0 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); - bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); - } else if (randVal == 3) { // 2, 1, 0 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); - } else if (randVal == 4) { // 2, 0, 1 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); - bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); - } - if (randVal == 5) { // 0, 2, 1 - bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); - } else { - } // 0, 1, 2 - } - + bnAdjustVnodeIndex(pVgroup); bnReleaseDnodes(); bnUnLock(); return TSDB_CODE_SUCCESS; @@ -186,17 +224,32 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { return false; } + int32_t rmVnodeVer = 0; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SVnodeGid *pVnode = pVgroup->vnodeGid + i; + if (pVnode == pRmVnode) { + rmVnodeVer = mnodeGetVgidVer(pVnode->vver); + mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i, + pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer); + } + } + bool isReady = false; for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVnode = pVgroup->vnodeGid + i; + SDnodeObj *pDnode = pVnode->pDnode; if (pVnode == pRmVnode) continue; + int32_t vver = mnodeGetVgidVer(pVnode->vver); - mTrace("vgId:%d, check vgroup status, dnode:%d status:%d, vnode role:%s", pVgroup->vgId, pVnode->pDnode->dnodeId, - pVnode->pDnode->status, syncRole[pVnode->role]); - if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue; - if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; + mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d", pVgroup->vgId, i, + pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); + if (pDnode->status == TAOS_DN_STATUS_DROPPING) continue; + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; + if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue; - if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) { + if (rmVnodeVer == 0 || vver >= rmVnodeVer) { + mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d", + pVgroup->vgId, i, pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); isReady = true; } } @@ -211,50 +264,14 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { static int32_t bnRemoveVnode(SVgObj *pVgroup) { if (pVgroup->numOfVnodes <= 1) return -1; - SVnodeGid *pRmVnode = NULL; - SVnodeGid *pSelVnode = NULL; - int32_t maxScore = 0; - - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pVnode = &(pVgroup->vnodeGid[i]); - SDnodeObj *pDnode = mnodeGetDnode(pVnode->dnodeId); - - if (pDnode == NULL) { - mError("vgId:%d, dnode:%d not exist, remove it", pVgroup->vgId, pVnode->dnodeId); - pRmVnode = pVnode; - break; - } - - if (pDnode->status == TAOS_DN_STATUS_DROPPING) { - mDebug("vgId:%d, dnode:%d in dropping state", pVgroup->vgId, pVnode->dnodeId); - pRmVnode = pVnode; - } else if (pVnode->dnodeId == pVgroup->lbDnodeId) { - mDebug("vgId:%d, dnode:%d in updating state", pVgroup->vgId, pVnode->dnodeId); - pRmVnode = pVnode; - } else { - if (pSelVnode == NULL) { - pSelVnode = pVnode; - maxScore = pDnode->score; - } else { - if (maxScore < pDnode->score) { - pSelVnode = pVnode; - maxScore = pDnode->score; - } - } - } - - mnodeDecDnodeRef(pDnode); - } - - if (pRmVnode != NULL) { - pSelVnode = pRmVnode; - } + SVnodeGid *pSelVnode = &pVgroup->vnodeGid[pVgroup->numOfVnodes - 1]; + mDebug("vgId:%d, vnode in dnode:%d will be dropped", pVgroup->vgId, pSelVnode->dnodeId); if (!bnCheckVgroupReady(pVgroup, pSelVnode)) { mDebug("vgId:%d, is not ready", pVgroup->vgId); return -1; } else { - mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); + mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); bnDiscardVnode(pVgroup, pSelVnode); return TSDB_CODE_SUCCESS; } @@ -272,36 +289,48 @@ static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) { return false; } -/** - * desc: add vnode to vgroup, find a new one if dest dnode is null - **/ -static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { - if (pDestDnode == NULL) { - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (pDnode == pSrcDnode) continue; - if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue; - if (!bnCheckFree(pDnode)) continue; - - pDestDnode = pDnode; - mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId); - break; - } +static SDnodeObj *bnGetAvailDnode(SVgObj *pVgroup) { + for (int32_t i = 0; i < tsBnDnodes.size; ++i) { + SDnodeObj *pDnode = tsBnDnodes.list[i]; + if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue; + if (!bnCheckFree(pDnode)) continue; + + mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId); + return pDnode; } - if (pDestDnode == NULL) { + return NULL; +} + +static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { + if (pDestDnode == NULL || pSrcDnode == pDestDnode) { return TSDB_CODE_MND_DNODE_NOT_EXIST; } - SVnodeGid *pVnodeGid = pVgroup->vnodeGid + pVgroup->numOfVnodes; - pVnodeGid->dnodeId = pDestDnode->dnodeId; - pVnodeGid->pDnode = pDestDnode; - pVgroup->numOfVnodes++; + SVnodeGid vnodeGids[TSDB_MAX_REPLICA]; + memcpy(&vnodeGids, &pVgroup->vnodeGid, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); + + int32_t numOfVnodes = pVgroup->numOfVnodes; + vnodeGids[numOfVnodes].dnodeId = pDestDnode->dnodeId; + vnodeGids[numOfVnodes].pDnode = pDestDnode; + numOfVnodes++; - if (pSrcDnode != NULL) { - pVgroup->lbDnodeId = pSrcDnode->dnodeId; + // move the src vnode to the end + for (int32_t v = 0; v < numOfVnodes; ++v) { + if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) { + bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]); + pVgroup->lbDnodeId = pSrcDnode->dnodeId; + break; + } } + // adjust the vgroup postion + if (pSrcDnode == NULL) { + bnAdjustVnodeIndex(pVgroup); + } + + memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); + pVgroup->numOfVnodes = numOfVnodes; atomic_add_fetch_32(&pDestDnode->openVnodes, 1); mnodeUpdateVgroup(pVgroup); @@ -312,16 +341,16 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes static bool bnMonitorBalance() { if (tsBnDnodes.size < 2) return false; + mDebug("monitor dnodes for balance, avail:%d", tsBnDnodes.size); for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) { SDnodeObj *pDnode = tsBnDnodes.list[src]; - mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1, - pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores, - pDnode->openVnodes); + mDebug("%d-dnode:%d, state:%s, score:%.1f, cores:%d, vnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId, + dnodeStatus[pDnode->status], pDnode->score, pDnode->numOfCores, pDnode->openVnodes); } float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score; if (scoresDiff < 0.01) { - mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff); + mDebug("all dnodes:%d is already balanced, scoreDiff:%.1f", tsBnDnodes.size, scoresDiff); return false; } @@ -389,7 +418,7 @@ void bnReset() { tsAccessSquence = 0; } -static int32_t bnMonitorVgroups() { +static bool bnMonitorVgroups() { void * pIter = NULL; SVgObj *pVgroup = NULL; bool hasUpdatingVgroup = false; @@ -409,7 +438,13 @@ static int32_t bnMonitorVgroups() { } else if (vgReplica < dbReplica) { mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica); hasUpdatingVgroup = true; - code = bnAddVnode(pVgroup, NULL, NULL); + + SDnodeObj *pAvailDnode = bnGetAvailDnode(pVgroup); + if (pAvailDnode == NULL) { + code = TSDB_CODE_MND_DNODE_NOT_EXIST; + } else { + code = bnAddVnode(pVgroup, NULL, pAvailDnode); + } } mnodeDecVgroupRef(pVgroup); @@ -529,6 +564,9 @@ void bnCheckStatus() { void * pIter = NULL; SDnodeObj *pDnode = NULL; + void *dnodeSdb = taosAcquireRef(tsSdbRid, tsDnodeRid); + if (dnodeSdb == NULL) return; + while (1) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; @@ -539,10 +577,13 @@ void bnCheckStatus() { mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence, pDnode->lastAccess, pDnode->status); bnSetVgroupOffline(pDnode); + bnStartTimer(3000); } } mnodeDecDnodeRef(pDnode); } + + taosReleaseRef(tsSdbRid, tsDnodeRid); } void bnCheckModules() { diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c index e5ad7a211958dce9507c5fca22f57a6d99c82029..7d94df1c23ab7824dbada0423beec14530a2101c 100644 --- a/src/balance/src/bnScore.c +++ b/src/balance/src/bnScore.c @@ -271,23 +271,23 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = systemScore; + *(float *)pWrite = (float)systemScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = pDnode->customScore; + *(float *)pWrite = (float)pDnode->customScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)moduleScore; + *(float *)pWrite = (float)moduleScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)vnodeScore; + *(float *)pWrite = (float)vnodeScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore); + *(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -299,7 +299,7 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status)); + STR_TO_VARSTR(pWrite, dnodeStatus[pDnode->status]); cols++; numOfRows++; diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index bf046a9faea6e9e3d7b324e7786c594d16a9aeb7..d07591ecd55a9a10613cc61de3a77ea85e022bc2 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -31,7 +31,10 @@ static void *bnThreadFunc(void *arg) { } pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); + mDebug("balance thread wakes up to work"); bool updateSoon = bnStart(); + mDebug("balance thread finished this poll, updateSoon:%d", updateSoon); + bnStartTimer(updateSoon ? 1000 : -1); pthread_mutex_unlock(&(tsBnThread.mutex)); } @@ -53,7 +56,7 @@ int32_t bnInitThread() { pthread_attr_destroy(&thattr); if (ret != 0) { - mError("failed to create balance thread since %s", strerror(errno)); + mError("failed to create balance thread since %s", strerror(ret)); return -1; } @@ -101,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) { tsBnThread.timer = NULL; tsAccessSquence++; - bnCheckStatus(); bnStartTimer(-1); + bnCheckStatus(); if (handle == NULL) { if (tsAccessSquence % tsBalanceInterval == 0) { @@ -116,12 +119,13 @@ static void bnProcessTimer(void *handle, void *tmrId) { } } -void bnStartTimer(int64_t mseconds) { +void bnStartTimer(int32_t mseconds) { if (tsBnThread.stop) return; bool updateSoon = (mseconds != -1); if (updateSoon) { - taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); + mTrace("balance function will be called after %d ms", mseconds); + taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); } diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index daf7c5e5349f42730721b3f2b6baad5d0485361e..fb43751b9e8fd715d538abb1198e1bdfd0a2e9ae 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -28,6 +28,28 @@ IF (TD_LINUX) ADD_SUBDIRECTORY(tests) +ELSEIF (TD_DARWIN) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) + + # set the static lib name + ADD_LIBRARY(taos_static STATIC ${SRC}) + TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") + SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + # generate dynamic library (*.dylib) + ADD_LIBRARY(taos SHARED ${SRC}) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + #set version of .dylib + #VERSION dylib version + #SOVERSION dylib version + #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") + SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) + + ADD_SUBDIRECTORY(tests) + ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) @@ -49,12 +71,12 @@ ELSEIF (TD_DARWIN) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") # generate dynamic library (*.dylib) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 43ba31f331fc5c92d4142a008db458b23395a473..581cd37cbd53cb87847fc5a13c88b03eb797d93a 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -38,13 +38,7 @@ typedef struct SLocalDataSource { tFilePage filePage; } SLocalDataSource; -enum { - TSC_LOCALREDUCE_READY = 0x0, - TSC_LOCALREDUCE_IN_PROGRESS = 0x1, - TSC_LOCALREDUCE_TOBE_FREED = 0x2, -}; - -typedef struct SLocalReducer { +typedef struct SLocalMerger { SLocalDataSource ** pLocalDataSrc; int32_t numOfBuffer; int32_t numOfCompleted; @@ -56,7 +50,6 @@ typedef struct SLocalReducer { tFilePage * pTempBuffer; struct SQLFunctionCtx *pCtx; int32_t rowSize; // size of each intermediate result. - int32_t status; // denote it is in reduce process, in reduce process, it bool hasPrevRow; // cannot be released bool hasUnprocessedRow; tOrderDescriptor * pDesc; @@ -69,7 +62,7 @@ typedef struct SLocalReducer { bool discard; int32_t offset; // limit offset value bool orderPrjOnSTable; // projection query on stable -} SLocalReducer; +} SLocalMerger; typedef struct SRetrieveSupport { tExtMemBuffer ** pExtMemBuffer; // for build loser tree @@ -96,10 +89,10 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF /* * create local reducer to launch the second-stage reduce process at client site */ -void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, +void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, SColumnModel *finalModel, SColumnModel *pFFModel, SSqlObj* pSql); -void tscDestroyLocalReducer(SSqlObj *pSql); +void tscDestroyLocalMerger(SSqlObj *pSql); int32_t tscDoLocalMerge(SSqlObj *pSql); diff --git a/src/client/inc/tscLog.h b/src/client/inc/tscLog.h index 9d01edae3680c2b6aa3c075229ef077eebbe230b..f25ec02bd8204d124477e6eba1070520b4d00c4e 100644 --- a/src/client/inc/tscLog.h +++ b/src/client/inc/tscLog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSC_LOG_H -#define TDENGINE_TSC_LOG_H +#ifndef TDENGINE_TSCLOG_H +#define TDENGINE_TSCLOG_H #ifdef __cplusplus extern "C" { @@ -22,16 +22,16 @@ extern "C" { #include "tlog.h" -extern int32_t cDebugFlag; -extern int32_t tscEmbedded; +extern uint32_t cDebugFlag; +extern int8_t tscEmbedded; -#define tscFatal(...) { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscError(...) { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscWarn(...) { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscInfo(...) { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscDebug(...) { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} -#define tscTrace(...) { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} -#define tscDebugL(...){ if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index f7832c9818226d19ba1e20cb47fa17c5bfb0f611..15ef54b7b13eb8463c54a7b979eb6d007560bb0e 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCJOINPROCESS_H -#define TDENGINE_TSCJOINPROCESS_H +#ifndef TDENGINE_TSCSUBQUERY_H +#define TDENGINE_TSCSUBQUERY_H #ifdef __cplusplus extern "C" { @@ -33,18 +33,23 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index); void tscHandleMasterJoinQuery(SSqlObj* pSql); int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); +int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql); int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); -int32_t tscHandleInsertRetry(SSqlObj* pSql); +int32_t tscHandleInsertRetry(SSqlObj* parent, SSqlObj* child); void tscBuildResFromSubqueries(SSqlObj *pSql); TAOS_ROW doSetResultRowData(SSqlObj *pSql); char *getArithmeticInputSrc(void *param, const char *name, int32_t colId); +void tscLockByThread(int64_t *lockedBy); + +void tscUnlockByThread(int64_t *lockedBy); + #ifdef __cplusplus } #endif -#endif // TDENGINE_TSCJOINPROCESS_H +#endif // TDENGINE_TSCSUBQUERY_H diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2c8641da760fd2e1f340cc4627559ec1a226686c..502d044d7540169497b682df96424ca3304f4668 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -20,9 +20,6 @@ extern "C" { #endif -/* - * @date 2018/09/30 - */ #include "exception.h" #include "os.h" #include "qExtbuffer.h" @@ -101,20 +98,20 @@ static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t sub return pCmd->pQueryInfo[subClauseIndex]; } -int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); +void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset); void* tscDestroyBlockArrayList(SArray* pDataBlockList); +void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta); + int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); -int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pDataList); -int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks); +int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap); +int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, SName* pName, STableMeta* pTableMeta, + STableDataBlocks** dataBlocks, SArray* pBlockList); /** * for the projection query on metric or point interpolation query on metric, @@ -135,8 +132,9 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); +bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); @@ -144,10 +142,6 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); -/* use for keep current db info temporarily, for handle table with db prefix */ -// todo remove it -void tscGetDBInfoFromTableFullName(char* tableId, char* db); - int tscAllocPayload(SSqlCmd* pCmd, int size); TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); @@ -181,6 +175,7 @@ SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIn SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); +void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); @@ -215,10 +210,10 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, + SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); @@ -227,13 +222,13 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscFreeVgroupTableInfo(SArray* pVgroupTables); -SArray* tscVgroupTableInfoClone(SArray* pVgroupTables); +SArray* tscVgroupTableInfoDup(SArray* pVgroupTables); void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); -int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); +int tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); void tscResetForNextRetrieve(SSqlRes* pRes); void tscDoQuery(SSqlObj* pSql); @@ -258,11 +253,11 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src); * @param pPrevSql * @return */ -SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd); +SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd); void registerSqlObj(SSqlObj* pSql); -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql); +SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql); void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex); void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); @@ -275,6 +270,8 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreClauseToTry(SSqlObj* pSql); +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta); + void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); @@ -284,6 +281,17 @@ bool tscSetSqlOwner(SSqlObj* pSql); void tscClearSqlOwner(SSqlObj* pSql); int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize); +char* serializeTagData(STagData* pTagData, char* pMsg); +int32_t copyTagData(STagData* dst, const STagData* src); + +STableMeta* createSuperTableMeta(STableMetaMsg* pChild); +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); +uint32_t tscGetTableMetaMaxSize(); +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name); +STableMeta* tscTableMetaDup(STableMeta* pTableMeta); + + void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); char* strdup_throw(const char* str); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index f6dc45398f35c38598f3f3132b2f6f5601a4ed68..a9dcd230a6db824f8cf40e05603c78bd95e0168b 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -24,10 +24,6 @@ extern "C" { #include "tstoken.h" #include "tsclient.h" -#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) - -//struct SSchema; - /** * get the number of tags of this table * @param pTableMeta @@ -79,33 +75,16 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex); */ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId); -/** - * check if the schema is valid or not, including following aspects: - * 1. number of columns - * 2. column types - * 3. column length - * 4. column names - * 5. total length - * - * @param pSchema - * @param numOfCols - * @return - */ -bool isValidSchema(struct SSchema *pSchema, int32_t numOfCols); - -/** - * get the schema for the "tbname" column. it is a built column - * @return - */ -SSchema tscGetTbnameColumnSchema(); - /** * create the table meta from the msg * @param pTableMetaMsg * @param size size of the table meta * @return */ -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg); + +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index a1b6174de0d21f02fdde1e367bcb45901707c6ba..0bfbf3f946a4a1428549a493ab990a2be2173266 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -22,63 +22,29 @@ extern "C" { #include "os.h" +#include "qAggMain.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" -#include "tglobal.h" -#include "tsqlfunction.h" -#include "tutil.h" #include "tcache.h" +#include "tglobal.h" #include "tref.h" +#include "tutil.h" #include "qExecutor.h" #include "qSqlparser.h" #include "qTsbuf.h" #include "tcmdtype.h" -#if 0 -static UNUSED_FUNC void *u_malloc (size_t __size) { - uint32_t v = rand(); - - if (v % 5000 <= 0) { - return NULL; - } else { - return malloc(__size); - } -} - -static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { - uint32_t v = rand(); - if (v % 5000 <= 0) { - return NULL; - } else { - return calloc(num, __size); - } -} - -static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { - uint32_t v = rand(); - if (v % 5000 <= 0) { - return NULL; - } else { - return realloc(p, __size); - } -} - -#define calloc u_calloc -#define malloc u_malloc -#define realloc u_realloc -#endif - // forward declaration struct SSqlInfo; -struct SLocalReducer; +struct SLocalMerger; // data source from sql string or from file enum { DATA_FROM_SQL_STRING = 1, - DATA_FROM_DATA_FILE = 2, + DATA_FROM_DATA_FILE = 2, }; typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); @@ -90,22 +56,28 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; -typedef struct SCorVgroupInfo { - int32_t version; - int8_t inUse; - int8_t numOfEps; - SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCorVgroupInfo; +typedef struct SNewVgroupInfo { + int32_t vgId; + int8_t inUse; + int8_t numOfEps; + SEpAddrMsg ep[TSDB_MAX_REPLICA]; +} SNewVgroupInfo; + +typedef struct CChildTableMeta { + int32_t vgId; + STableId id; + uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name +} CChildTableMeta; typedef struct STableMeta { - STableComInfo tableInfo; + int32_t vgId; + STableId id; uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; int16_t sversion; int16_t tversion; - char sTableId[TSDB_TABLE_FNAME_LEN]; - SVgroupInfo vgroupInfo; - SCorVgroupInfo corVgroupInfo; - STableId id; + STableComInfo tableInfo; SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; @@ -118,10 +90,10 @@ typedef struct STableMetaInfo { * 1. keep the vgroup index during the multi-vnode super table projection query * 2. keep the vgroup index for multi-vnode insertion */ - int32_t vgroupIndex; - char name[TSDB_TABLE_FNAME_LEN]; // (super) table name - char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql - SArray* tagColList; // SArray, involved tag columns + int32_t vgroupIndex; + SName name; + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray *tagColList; // SArray, involved tag columns } STableMetaInfo; /* the structure for sql function in select clause */ @@ -136,7 +108,7 @@ typedef struct SSqlExpr { int16_t numOfParams; // argument value of each function tVariant param[3]; // parameters are not more than 3 int32_t offset; // sub result column value of arithmetic expression. - int16_t resColId; // result column id + int16_t resColId; // result column id } SSqlExpr; typedef struct SColumnIndex { @@ -170,7 +142,7 @@ typedef struct SCond { } SCond; typedef struct SJoinNode { - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableName[TSDB_TABLE_FNAME_LEN]; uint64_t uid; int16_t tagColId; } SJoinNode; @@ -204,22 +176,17 @@ typedef struct SParamInfo { } SParamInfo; typedef struct STableDataBlocks { - char tableId[TSDB_TABLE_FNAME_LEN]; - int8_t tsSource; // where does the UNIX timestamp come from, server or client - bool ordered; // if current rows are ordered or not - int64_t vgId; // virtual group id - int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending - int32_t numOfTables; // number of tables in current submit block - int32_t rowSize; // row size for current table - uint32_t nAllocSize; - uint32_t headerSize; // header for table info (uid, tid, submit metadata) - uint32_t size; - - /* - * the table meta of table, the table meta will be used during submit, keep a ref - * to avoid it to be removed from cache - */ - STableMeta *pTableMeta; + SName tableName; + int8_t tsSource; // where does the UNIX timestamp come from, server or client + bool ordered; // if current rows are ordered or not + int64_t vgId; // virtual group id + int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending + int32_t numOfTables; // number of tables in current submit block + int32_t rowSize; // row size for current table + uint32_t nAllocSize; + uint32_t headerSize; // header for table info (uid, tid, submit metadata) + uint32_t size; + STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; // for parameter ('?') binding @@ -252,10 +219,14 @@ typedef struct SQueryInfo { int64_t clauseLimit; // limit for current sub clause int64_t prjOffset; // offset value in the original sql expression, only applied at client side - int64_t tableLimit; // table limit in case of super table projection query + global order + limit + int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX int16_t resColumnId; // result column id + bool distinctTag; // distinct tag or not + int32_t round; // 0/1/.... + int32_t bufLen; + char* buf; } SQueryInfo; typedef struct { @@ -268,7 +239,7 @@ typedef struct { int32_t numOfTablesInSubmit; }; - int32_t insertType; + uint32_t insertType; // TODO remove it int32_t clauseIndex; // index of multiple subclause query char * curSql; // current sql, resume position of sql after parsing paused @@ -284,10 +255,14 @@ typedef struct { int32_t numOfParams; int8_t dataSourceType; // load data from file or not - int8_t submitSchema; // submit block is built with table schema - STagData *pTagData; // NOTE: pTagData->data is used as a variant length array - SHashObj *pTableList; // referred table involved in sql - SArray *pDataBlocks; // SArray submit data blocks after parsing sql + int8_t submitSchema; // submit block is built with table schema + STagData tagData; // NOTE: pTagData->data is used as a variant length array + + SName **pTableNameList; // all involved tableMeta list of current insert sql statement. + int32_t numOfTables; + + SHashObj *pTableBlockHashList; // data block for each table + SArray *pDataBlocks; // SArray. Merged submit block for each vgroup } SSqlCmd; typedef struct SResRec { @@ -320,17 +295,22 @@ typedef struct { char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex* pColumnIndex; - SArithmeticSupport* pArithSup; // support the arithmetic expression calculation on agg functions - struct SLocalReducer* pLocalReducer; + SArithmeticSupport *pArithSup; // support the arithmetic expression calculation on agg functions + struct SLocalMerger *pLocalMerger; } SSqlRes; +typedef struct { + char key[512]; + void *pDnodeConn; +} SRpcObj; + typedef struct STscObj { void * signature; void * pTimer; char user[TSDB_USER_LEN]; char pass[TSDB_KEY_LEN]; - char acctId[TSDB_ACCT_LEN]; - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; char sversion[TSDB_VERSION_LEN]; char writeAuth : 1; char superAuth : 1; @@ -339,25 +319,26 @@ typedef struct STscObj { int64_t hbrid; struct SSqlObj * sqlList; struct SSqlStream *streamList; - SRpcCorEpSet *tscCorMgmtEpSet; - void* pDnodeConn; + SRpcObj *pRpcObj; + SRpcCorEpSet *tscCorMgmtEpSet; pthread_mutex_t mutex; - T_REF_DECLARE() + int32_t numOfObj; // number of sqlObj from this tscObj } STscObj; typedef struct SSubqueryState { - int32_t numOfRemain; // the number of remain unfinished subquery + pthread_mutex_t mutex; + int8_t *states; int32_t numOfSub; // the number of total sub-queries uint64_t numOfRetrievedRows; // total number of points in this query } SSubqueryState; typedef struct SSqlObj { void *signature; - pthread_t owner; // owner of sql object, by which it is executed + int64_t owner; // owner of sql object, by which it is executed STscObj *pTscObj; int64_t rpcRid; - void (*fp)(); - void (*fetchFp)(); + __async_cb_func_t fp; + __async_cb_func_t fetchFp; void *param; int64_t stime; uint32_t queryId; @@ -376,12 +357,18 @@ typedef struct SSqlObj { SSubqueryState subState; struct SSqlObj **pSubs; + int64_t metaRid; + int64_t svgroupRid; + + int64_t squeryLock; + struct SSqlObj *prev, *next; int64_t self; } SSqlObj; typedef struct SSqlStream { SSqlObj *pSql; + const char* dstTable; uint32_t streamId; char listed; bool isProject; @@ -408,8 +395,12 @@ typedef struct SSqlStream { struct SSqlStream *prev, *next; } SSqlStream; -int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn); -void tscInitMsgsFp(); +void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable); + + +int tscAcquireRpc(const char *key, const char *user, const char *secret,void **pRpcObj); +void tscReleaseRpc(void *param); +void tscInitMsgsFp(); int tsParseSql(SSqlObj *pSql, bool initial); @@ -417,21 +408,20 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); int tscProcessSql(SSqlObj *pSql); int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex); -void tscQueueAsyncRes(SSqlObj *pSql); +void tscAsyncResultOnError(SSqlObj *pSql); void tscQueueAsyncError(void(*fp), void *param, int32_t code); int tscProcessLocalCmd(SSqlObj *pSql); int tscCfgDynamicOptions(char *msg); -int taos_retrieve(TAOS_RES *res); -int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); -void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); +int32_t tscTansformFuncForSTableQuery(SQueryInfo *pQueryInfo); +void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); +void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta); /** * free query result of the sql object @@ -445,21 +435,25 @@ void tscFreeSqlResult(SSqlObj *pSql); */ void tscFreeSqlObj(SSqlObj *pSql); void tscFreeRegisteredSqlObj(void *pSql); -void tscFreeTableMetaHelper(void *pTableMeta); void tscCloseTscObj(void *pObj); // todo move to taos? or create a new file: taos_internal.h TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos); -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res); +TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res); +TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param); + void waitForQueryRsp(void *param, TAOS_RES *tres, int code); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen); -void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql); +void tscImportDataFromFile(SSqlObj *pSql); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); bool tscIsUpdateQuery(SSqlObj* pSql); +char* tscGetSqlStr(SSqlObj* pSql); +bool tscIsQueryWithLimit(SSqlObj* pSql); + bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); @@ -485,9 +479,9 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : (unsigned char*)pData; } else { - assert(bytes == tDataTypeDesc[type].nSize); + assert(bytes == tDataTypes[type].bytes); - pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)&pInfo->pSqlExpr->param[1].i64Key; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)&pInfo->pSqlExpr->param[1].i64; pRes->length[columnIndex] = bytes; } } else { @@ -502,7 +496,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField pRes->length[columnIndex] = realLen; } else { - assert(bytes == tDataTypeDesc[type].nSize); + assert(bytes == tDataTypes[type].bytes); pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)pData; pRes->length[columnIndex] = bytes; @@ -510,15 +504,16 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } } -extern SCacheObj* tscMetaCache; -extern int tscObjRef; -extern void * tscTmr; -extern void * tscQhandle; -extern int tscKeepConn[]; -extern int tsInsertHeadSize; -extern int tscNumOfThreads; -extern int tscRefId; - +extern int32_t sentinel; +extern SHashObj *tscVgroupMap; +extern SHashObj *tscTableMetaInfo; + +extern int tscObjRef; +extern void *tscTmr; +extern void *tscQhandle; +extern int tscKeepConn[]; +extern int tscRefId; +extern int tscNumOfObj; // number of existed sqlObj in current process. extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/taos.def b/src/client/src/taos.def index 49d7290ce7bbb9cf64bd71c8d263ad3913576a71..43cd8190614facc207bbad0885360f23c8216c8a 100644 --- a/src/client/src/taos.def +++ b/src/client/src/taos.def @@ -32,7 +32,6 @@ taos_errstr taos_errno taos_query_a taos_fetch_rows_a -taos_fetch_row_a taos_subscribe taos_consume taos_unsubscribe diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 3ff8a68d8f07f776dbb03b4b55b830b7e37dfdff..5cba897b3028a2d6301e9363ca6180f6b41022f8 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -18,27 +18,21 @@ #include "tnote.h" #include "trpc.h" -#include "tcache.h" #include "tscLog.h" #include "tscSubquery.h" -#include "tscLocalMerge.h" #include "tscUtil.h" #include "tsched.h" #include "tschemautil.h" #include "tsclient.h" -static void tscProcessFetchRow(SSchedMsg *pMsg); static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); -static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, void (*fp)()); - /* * Proxy function to perform sequentially query&retrieve operation. * If sql queries upon a super table and two-stage merge procedure is not involved (when employ the projection * query), it will sequentially query&retrieve data for all vnodes */ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); -static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) { SSqlCmd* pCmd = &pSql->cmd; @@ -57,7 +51,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -71,7 +65,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -80,12 +74,16 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para // TODO return the correct error code to client in tscQueueAsyncError void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { + taos_query_ra(taos, sqlstr, fp, param); +} + +TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { tscError("bug!!! pObj:%p", pObj); terrno = TSDB_CODE_TSC_DISCONNECTED; tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return; + return NULL; } int32_t sqlLen = (int32_t)strlen(sqlstr); @@ -93,21 +91,24 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; tscQueueAsyncError(fp, param, terrno); - return; + return NULL; } - taosNotePrintTsc(sqlstr); + nPrintTsc("%s", sqlstr); SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { tscError("failed to malloc sqlObj"); tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY); - return; + return NULL; } doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen); + + return pSql; } + static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { if (tres == NULL) { return; @@ -149,7 +150,7 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { } // actual continue retrieve function with user-specified callback function -static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, void (*fp)()) { +static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, __async_cb_func_t fp) { SSqlObj *pSql = (SSqlObj *)tres; if (pSql == NULL) { // error tscError("sql object is NULL"); @@ -166,7 +167,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pRes->code = numOfRows; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -192,11 +193,6 @@ static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOf tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchRowsProxy); } -void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows) { - // query completed, continue to retrieve - tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchSingleRowProxy); -} - void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { SSqlObj *pSql = (SSqlObj *)taosa; if (pSql == NULL || pSql->signature != pSql) { @@ -217,7 +213,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; pSql->param = param; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -264,115 +260,19 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { } } -void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), void *param) { - SSqlObj *pSql = (SSqlObj *)taosa; - if (pSql == NULL || pSql->signature != pSql) { - tscError("sql object is NULL"); - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return; - } - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - if (pRes->qhandle == 0) { - tscError("qhandle is NULL"); - pSql->param = param; - pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - - tscQueueAsyncRes(pSql); - return; - } - - pSql->fetchFp = fp; - pSql->param = param; - - if (pRes->row >= pRes->numOfRows) { - tscResetForNextRetrieve(pRes); - pSql->fp = tscAsyncFetchSingleRowProxy; - - if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); - } else { - SSchedMsg schedMsg = { 0 }; - schedMsg.fp = tscProcessFetchRow; - schedMsg.ahandle = pSql; - schedMsg.thandle = pRes->tsrow; - schedMsg.msg = NULL; - taosScheduleTask(tscQhandle, &schedMsg); - } -} - -void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) { - SSqlObj *pSql = (SSqlObj *)tres; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - if (numOfRows == 0) { - if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. - tscTryQueryNextVnode(pSql, tscAsyncQuerySingleRowForNextVnode); - } else { - /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore - */ - (*pSql->fetchFp)(pSql->param, pSql, NULL); - } - return; - } - - for (int i = 0; i < pCmd->numOfCols; ++i){ - SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (pSup->pSqlExpr != NULL) { -// pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row; - } else { - //todo add - } - } - - pRes->row++; - - (*pSql->fetchFp)(pSql->param, pSql, pSql->res.tsrow); -} - -void tscProcessFetchRow(SSchedMsg *pMsg) { - SSqlObj *pSql = (SSqlObj *)pMsg->ahandle; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - for (int i = 0; i < pCmd->numOfCols; ++i) { - SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - - if (pSup->pSqlExpr != NULL) { - tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, 0); - } else { -// todo add - } - } - - pRes->row++; - (*pSql->fetchFp)(pSql->param, pSql, pRes->tsrow); -} - // this function will be executed by queue task threads, so the terrno is not valid static void tscProcessAsyncError(SSchedMsg *pMsg) { void (*fp)() = pMsg->ahandle; terrno = *(int32_t*) pMsg->msg; - (*fp)(pMsg->thandle, NULL, *(int32_t*)pMsg->msg); + tfree(pMsg->msg); + (*fp)(pMsg->thandle, NULL, terrno); } void tscQueueAsyncError(void(*fp), void *param, int32_t code) { int32_t* c = malloc(sizeof(int32_t)); *c = code; - SSchedMsg schedMsg = { 0 }; + SSchedMsg schedMsg = {0}; schedMsg.fp = tscProcessAsyncError; schedMsg.ahandle = fp; schedMsg.thandle = param; @@ -380,26 +280,42 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) { taosScheduleTask(tscQhandle, &schedMsg); } -void tscQueueAsyncRes(SSqlObj *pSql) { +static void tscAsyncResultCallback(SSchedMsg *pMsg) { + SSqlObj* pSql = pMsg->ahandle; if (pSql == NULL || pSql->signature != pSql) { tscDebug("%p SqlObj is freed, not add into queue async res", pSql); return; } - tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code)); + assert(pSql->res.code != TSDB_CODE_SUCCESS); + tscError("%p invoke user specified function due to error occurred, code:%s", pSql, tstrerror(pSql->res.code)); SSqlRes *pRes = &pSql->res; - assert(pSql->fp != NULL && pSql->fetchFp != NULL); + if (pSql->fp == NULL || pSql->fetchFp == NULL){ + return; + } pSql->fp = pSql->fetchFp; (*pSql->fp)(pSql->param, pSql, pRes->code); } +void tscAsyncResultOnError(SSqlObj* pSql) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = tscAsyncResultCallback; + schedMsg.ahandle = pSql; + schedMsg.thandle = (void *)1; + schedMsg.msg = 0; + taosScheduleTask(tscQhandle, &schedMsg); +} + + int tscSendMsgToServer(SSqlObj *pSql); void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { - SSqlObj *pSql = (SSqlObj *)param; - if (pSql == NULL || pSql->signature != pSql) return; + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param); + if (pSql == NULL) return; + + assert(pSql->signature == pSql && (int64_t)param == pSql->self); SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; @@ -410,102 +326,72 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code != TSDB_CODE_SUCCESS) { tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code)); goto _error; - } else { - tscDebug("%p get %s successfully", pSql, msg); } + tscDebug("%p get %s successfully", pSql, msg); if (pSql->pStream == NULL) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // check if it is a sub-query of super table query first, if true, enter another routine - if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY)) { - tscDebug("%p update table meta in local cache, continue to process sql and send corresponding subquery", pSql); + if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) { + tscDebug("%p update local table meta, continue to process sql and send the corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return; - } else { - assert(code == TSDB_CODE_SUCCESS); - } - - // param already freed by other routine and pSql in tscCache when ctrl + c - if (atomic_load_ptr(&pSql->param) == NULL) { - return; - } - assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); - - SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; - SSqlObj * pParObj = trs->pParentSql; - - // NOTE: the vgroupInfo for the queried super table must be existed here. - assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && - pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL); + assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS || code == TSDB_CODE_SUCCESS); - // tscProcessSql can add error into async res - tscProcessSql(pSql); - return; - } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) { - tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - code = tscGetTableMeta(pSql, pTableMetaInfo); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); return; - } else { - assert(code == TSDB_CODE_SUCCESS); } assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); + // tscProcessSql can add error into async res tscProcessSql(pSql); + taosReleaseRef(tscObjRef, pSql->self); return; } else { // continue to process normal async query if (pCmd->parseFinished) { - tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql); + tscDebug("%p update local table meta, continue to process sql and send corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); + + assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS || code == TSDB_CODE_SUCCESS); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); return; - } else { - assert(code == TSDB_CODE_SUCCESS); } - // in case of insert, redo parsing the sql string and build new submit data block for two reasons: - // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. - // 2. vnode may need the schema information along with submit block to update its local table schema. - if (pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_SELECT) { + assert(pCmd->command != TSDB_SQL_INSERT); + + if (pCmd->command == TSDB_SQL_SELECT) { tscDebug("%p redo parse sql string and proceed", pSql); pCmd->parseFinished = false; - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmd(pCmd, true); code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); return; } else if (code != TSDB_CODE_SUCCESS) { goto _error; } - if (pCmd->command == TSDB_SQL_INSERT) { - /* - * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks, - * and send the required submit block according to index value in supporter to server. - */ - pSql->fp = pSql->fetchFp; // restore the fp - tscHandleInsertRetry(pSql); - } else if (pCmd->command == TSDB_SQL_SELECT) { // in case of other query type, continue - tscProcessSql(pSql); - } - }else { // in all other cases, simple retry + tscProcessSql(pSql); + } else { // in all other cases, simple retry tscProcessSql(pSql); } + taosReleaseRef(tscObjRef, pSql->self); return; } else { tscDebug("%p continue parse sql after get table meta", pSql); code = tsParseSql(pSql, false); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); return; } else if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -515,12 +401,14 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); return; } else { assert(code == TSDB_CODE_SUCCESS); } (*pSql->fp)(pSql->param, pSql, code); + taosReleaseRef(tscObjRef, pSql->self); return; } @@ -533,6 +421,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { code = tscGetTableMeta(pSql, pTableMetaInfo); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); return; } else if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -541,6 +430,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, pSql->self); return; } else if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -551,17 +441,25 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (!pSql->cmd.parseFinished) { tsParseSql(pSql, false); } + (*pSql->fp)(pSql->param, pSql, code); + + taosReleaseRef(tscObjRef, pSql->self); return; } tscDoQuery(pSql); + + taosReleaseRef(tscObjRef, pSql->self); + return; _error: if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } + + taosReleaseRef(tscObjRef, pSql->self); } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 4c28adc261f6b83b3c1bbb68d3dd3d56e6007fc9..820572859e88540656b8ce42dc3a6e77467c1423 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -17,7 +17,6 @@ #include "taosmsg.h" #include "taosdef.h" -#include "tcache.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" @@ -46,7 +45,8 @@ typedef struct SCreateBuilder { SSqlObj *pInterSql; int32_t (*fp)(void *para, char* result); Stage callStage; -} SCreateBuilder; +} SCreateBuilder; + static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { @@ -79,7 +79,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, pField->bytes); - char *type = tDataTypeDesc[pSchema[i].type].aName; + char *type = tDataTypes[pSchema[i].type].name; pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; @@ -119,7 +119,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { // type name pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - char *type = tDataTypeDesc[pSchema[i].type].aName; + char *type = tDataTypes[pSchema[i].type].name; output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; STR_WITH_MAXSIZE_TO_VARSTR(output, type, pField->bytes); @@ -204,13 +204,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { assert(tscGetMetaInfo(pQueryInfo, 0)->pTableMeta != NULL); const int32_t NUM_OF_DESC_TABLE_COLUMNS = 4; - const int32_t TYPE_COLUMN_LENGTH = 16; + const int32_t TYPE_COLUMN_LENGTH = 20; const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql); -// if (noteFieldLen == 0) { -// noteFieldLen = NOTE_COLUMN_MIN_LENGTH; -// } + int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH; int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); tscFieldInfoUpdateOffset(pQueryInfo); @@ -275,7 +272,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code != TSDB_CODE_SUCCESS) { taos_free_result(pSql); free(builder); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -293,7 +290,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, code); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -571,11 +568,13 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch return TSDB_CODE_TSC_OUT_OF_MEMORY; } - char fullName[TSDB_TABLE_FNAME_LEN] = {0}; - extractDBName(pTableMetaInfo->name, fullName); - extractTableName(pMeta->sTableId, param->sTableName); + char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0}; + tNameGetDbName(&pTableMetaInfo->name, fullName); + + extractTableName(pMeta->sTableName, param->sTableName); snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName); - extractTableName(pTableMetaInfo->name, param->buf); + + strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN); param->pParentSql = pSql; param->pInterSql = pInterSql; @@ -605,6 +604,7 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch return TSDB_CODE_TSC_ACTION_IN_PROGRESS; } + static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, char *ddl) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -622,9 +622,9 @@ static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName, bytes); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypes[pSchema[i].type].name, bytes); } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[pSchema[i].type].name); } } sprintf(result + strlen(result) - 1, "%s", ")"); @@ -649,9 +649,9 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); } } snprintf(result + strlen(result) - 1, TSDB_MAX_BINARY_LEN - strlen(result), "%s %s", ")", "TAGS ("); @@ -663,9 +663,9 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); } } sprintf(result + strlen(result) - 1, "%s", ")"); @@ -678,8 +678,7 @@ static int32_t tscProcessShowCreateTable(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); assert(pTableMetaInfo->pTableMeta != NULL); - char tableName[TSDB_TABLE_NAME_LEN] = {0}; - extractTableName(pTableMetaInfo->name, tableName); + const char* tableName = tNameGetTableName(&pTableMetaInfo->name); char *result = (char *)calloc(1, TSDB_MAX_BINARY_LEN); int32_t code = TSDB_CODE_SUCCESS; @@ -715,7 +714,9 @@ static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) { free(pInterSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - extractTableName(pTableMetaInfo->name, param->buf); + + strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN); + param->pParentSql = pSql; param->pInterSql = pInterSql; param->fp = tscRebuildCreateDBStatement; @@ -748,7 +749,10 @@ static int32_t tscProcessCurrentUser(SSqlObj *pSql) { static int32_t tscProcessCurrentDB(SSqlObj *pSql) { char db[TSDB_DB_NAME_LEN] = {0}; + + pthread_mutex_lock(&pSql->pTscObj->mutex); extractDBName(pSql->pTscObj->db, db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); @@ -822,26 +826,39 @@ static int32_t tscProcessClientVer(SSqlObj *pSql) { } +// TODO add test cases. +static int32_t checkForOnlineNode(SSqlObj* pSql) { + int32_t* data = pSql->res.length; + if (data == NULL) { + return TSDB_CODE_SUCCESS; + } + + int32_t total = data[0]; + int32_t online = data[1]; + return (online < total)? TSDB_CODE_RPC_NETWORK_UNAVAIL:TSDB_CODE_SUCCESS; +} + static int32_t tscProcessServStatus(SSqlObj *pSql) { STscObj* pObj = pSql->pTscObj; SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); if (pHb != NULL) { - int32_t code = pHb->res.code; + pSql->res.code = pHb->res.code; taosReleaseRef(tscObjRef, pObj->hbrid); - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return pSql->res.code; - } - } else { - if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - return pSql->res.code; - } } - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return pSql->res.code; + } + + pSql->res.code = checkForOnlineNode(pHb); + if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return pSql->res.code; + } + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); + int32_t val = 1; tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t)); return TSDB_CODE_SUCCESS; @@ -875,7 +892,12 @@ int tscProcessLocalCmd(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; if (pCmd->command == TSDB_SQL_CFG_LOCAL) { - pRes->code = (uint8_t)taosCfgDynamicOptions(pCmd->payload); + if (taosCfgDynamicOptions(pCmd->payload)) { + pRes->code = TSDB_CODE_SUCCESS; + } else { + pRes->code = TSDB_CODE_COM_INVALID_CFG_MSG; + } + pRes->numOfRows = 0; } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) { pRes->code = (uint8_t)tscProcessDescribeTable(pSql); } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { @@ -890,7 +912,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { pRes->code = tscProcessShowCreateDatabase(pSql); } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosCacheEmpty(tscMetaCache); + taosHashEmpty(tscTableMetaInfo); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); @@ -914,7 +936,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { (*pSql->fp)(pSql->param, pSql, code); } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){ } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 9fdadfa957f3e6662ece8e0ee254508cda192ded..23fb0ab67cded77ff737fac4246343486e80eb95 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -16,7 +16,7 @@ #include "tscLocalMerge.h" #include "tscSubquery.h" #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "tlosertree.h" #include "tscLog.h" #include "tscUtil.h" @@ -56,7 +56,7 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { } } -static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDescriptor *pDesc) { +static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescriptor *pDesc) { /* * the fields and offset attributes in pCmd and pModel may be different due to * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. @@ -68,7 +68,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); - pCtx->aOutputBuf = pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; + pCtx->pOutput = pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; pCtx->order = pQueryInfo->order.order; pCtx->functionId = pExpr->functionId; @@ -76,7 +76,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc int16_t offset = getColumnModelOffset(pDesc->pColumnModel, i); SSchema *pSchema = getColumnModelSchema(pDesc->pColumnModel, i); - pCtx->aInputElemBuf = pReducer->pTempBuffer->data + offset; + pCtx->pInput = pReducer->pTempBuffer->data + offset; // input data format comes from pModel pCtx->inputType = pSchema->type; @@ -86,20 +86,19 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->outputBytes = pExpr->resBytes; pCtx->outputType = pExpr->resType; - pCtx->startOffset = 0; pCtx->size = 1; pCtx->hasNull = true; - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; // for top/bottom function, the output of timestamp is the first column int32_t functionId = pExpr->functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; - pCtx->param[2].i64Key = pQueryInfo->order.order; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pReducer->pCtx[0].pOutput; + pCtx->param[2].i64 = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[1].i64Key = pQueryInfo->order.orderColId; + pCtx->param[1].i64 = pQueryInfo->order.orderColId; } else if (functionId == TSDB_FUNC_APERCT) { - pCtx->param[0].i64Key = pExpr->param[0].i64Key; + pCtx->param[0].i64 = pExpr->param[0].i64; pCtx->param[0].nType = pExpr->param[0].nType; } @@ -118,7 +117,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExpr->resBytes; pTagCtx[n++] = &pReducer->pCtx[i]; - } else if ((aAggs[pExpr->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + } else if ((aAggs[pExpr->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { pCtx = &pReducer->pCtx[i]; } } @@ -166,7 +165,7 @@ static SFillColInfo* createFillColInfo(SQueryInfo* pQueryInfo) { return pFillCol; } -void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, +void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, SColumnModel *finalmodel, SColumnModel *pFFModel, SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; @@ -212,9 +211,9 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd return; } - size_t size = sizeof(SLocalReducer) + POINTER_BYTES * numOfFlush; + size_t size = sizeof(SLocalMerger) + POINTER_BYTES * numOfFlush; - SLocalReducer *pReducer = (SLocalReducer *) calloc(1, size); + SLocalMerger *pReducer = (SLocalMerger *) calloc(1, size); if (pReducer == NULL) { tscError("%p failed to create local merge structure, out of memory", pSql); @@ -311,7 +310,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->pCtx = (SQLFunctionCtx *)calloc(tscSqlExprNumOfExprs(pQueryInfo), sizeof(SQLFunctionCtx)); pReducer->rowSize = pMemBuffer[0]->nElemSize; - tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscRestoreFuncForSTableQuery(pQueryInfo); tscFieldInfoUpdateOffset(pQueryInfo); if (pReducer->rowSize > pMemBuffer[0]->pageSize) { @@ -372,7 +371,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->offset = (int32_t)pQueryInfo->limit.offset; - pRes->pLocalReducer = pReducer; + pRes->pLocalMerger = pReducer; pRes->numOfGroups = 0; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); @@ -383,7 +382,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pQueryInfo->fillType != TSDB_FILL_NONE) { SFillColInfo* pFillCol = createFillColInfo(pQueryInfo); - pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, + pReducer->pFillInfo = taosCreateFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, 4096, (int32_t)pQueryInfo->fieldsInfo.numOfOutput, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit, tinfo.precision, pQueryInfo->fillType, pFillCol, pSql); } @@ -477,13 +476,13 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa return 0; } -void tscDestroyLocalReducer(SSqlObj *pSql) { +void tscDestroyLocalMerger(SSqlObj *pSql) { if (pSql == NULL) { return; } SSqlRes *pRes = &(pSql->res); - if (pRes->pLocalReducer == NULL) { + if (pRes->pLocalMerger == NULL) { return; } @@ -491,21 +490,14 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // there is no more result, so we release all allocated resource - SLocalReducer *pLocalReducer = (SLocalReducer *)atomic_exchange_ptr(&pRes->pLocalReducer, NULL); - if (pLocalReducer != NULL) { - int32_t status = 0; - while ((status = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, - TSC_LOCALREDUCE_TOBE_FREED)) == TSC_LOCALREDUCE_IN_PROGRESS) { - taosMsleep(100); - tscDebug("%p waiting for delete procedure, status: %d", pSql, status); - } - - pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo); + SLocalMerger *pLocalMerge = (SLocalMerger *)atomic_exchange_ptr(&pRes->pLocalMerger, NULL); + if (pLocalMerge != NULL) { + pLocalMerge->pFillInfo = taosDestroyFillInfo(pLocalMerge->pFillInfo); - if (pLocalReducer->pCtx != NULL) { + if (pLocalMerge->pCtx != NULL) { int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[i]; tVariantDestroy(&pCtx->tag); tfree(pCtx->resultInfo); @@ -515,31 +507,31 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { } } - tfree(pLocalReducer->pCtx); + tfree(pLocalMerge->pCtx); } - tfree(pLocalReducer->prevRowOfInput); + tfree(pLocalMerge->prevRowOfInput); - tfree(pLocalReducer->pTempBuffer); - tfree(pLocalReducer->pResultBuf); + tfree(pLocalMerge->pTempBuffer); + tfree(pLocalMerge->pResultBuf); - if (pLocalReducer->pLoserTree) { - tfree(pLocalReducer->pLoserTree->param); - tfree(pLocalReducer->pLoserTree); + if (pLocalMerge->pLoserTree) { + tfree(pLocalMerge->pLoserTree->param); + tfree(pLocalMerge->pLoserTree); } - tfree(pLocalReducer->pFinalRes); - tfree(pLocalReducer->discardData); + tfree(pLocalMerge->pFinalRes); + tfree(pLocalMerge->discardData); - tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel, pLocalReducer->finalModel, - pLocalReducer->numOfVnode); - for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) { - tfree(pLocalReducer->pLocalDataSrc[i]); + tscLocalReducerEnvDestroy(pLocalMerge->pExtMemBuffer, pLocalMerge->pDesc, pLocalMerge->resColModel, pLocalMerge->finalModel, + pLocalMerge->numOfVnode); + for (int32_t i = 0; i < pLocalMerge->numOfBuffer; ++i) { + tfree(pLocalMerge->pLocalDataSrc[i]); } - pLocalReducer->numOfBuffer = 0; - pLocalReducer->numOfCompleted = 0; - free(pLocalReducer); + pLocalMerge->numOfBuffer = 0; + pLocalMerge->numOfCompleted = 0; + free(pLocalMerge); } else { tscDebug("%p already freed or another free function is invoked", pSql); } @@ -611,7 +603,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } } -bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage *tmpBuffer) { +bool isSameGroup(SSqlCmd *pCmd, SLocalMerger *pReducer, char *pPrev, tFilePage *tmpBuffer) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // disable merge procedure for column projection query @@ -726,10 +718,14 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSchema p1 = {0}; - if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) { - p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); + if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { + p1 = *tGetTbnameColumnSchema(); + } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { + p1.bytes = pExpr->resBytes; + p1.type = (uint8_t) pExpr->resType; + tstrncpy(p1.name, pExpr->aliasName, tListLen(p1.name)); } else { - p1 = tGetTableNameColumnSchema(); + p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); } int32_t inter = 0; @@ -747,6 +743,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr functionId = TSDB_FUNC_FIRST; } else if (functionId == TSDB_FUNC_LAST_DST) { functionId = TSDB_FUNC_LAST; + } else if (functionId == TSDB_FUNC_STDDEV_DST) { + functionId = TSDB_FUNC_STDDEV; } int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false); @@ -798,12 +796,12 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe /** * - * @param pLocalReducer + * @param pLocalMerge * @param pOneInterDataSrc * @param treeList * @return the number of remain input source. if ret == 0, all data has been handled */ -int32_t loadNewDataFromDiskFor(SLocalReducer *pLocalReducer, SLocalDataSource *pOneInterDataSrc, +int32_t loadNewDataFromDiskFor(SLocalMerger *pLocalMerge, SLocalDataSource *pOneInterDataSrc, bool *needAdjustLoserTree) { pOneInterDataSrc->rowIdx = 0; pOneInterDataSrc->pageId += 1; @@ -820,17 +818,17 @@ int32_t loadNewDataFromDiskFor(SLocalReducer *pLocalReducer, SLocalDataSource *p #endif *needAdjustLoserTree = true; } else { - pLocalReducer->numOfCompleted += 1; + pLocalMerge->numOfCompleted += 1; pOneInterDataSrc->rowIdx = -1; pOneInterDataSrc->pageId = -1; *needAdjustLoserTree = true; } - return pLocalReducer->numOfBuffer; + return pLocalMerge->numOfBuffer; } -void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *pOneInterDataSrc, +void adjustLoserTreeFromNewData(SLocalMerger *pLocalMerge, SLocalDataSource *pOneInterDataSrc, SLoserTreeInfo *pTree) { /* * load a new data page into memory for intermediate dataset source, @@ -838,7 +836,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * */ bool needToAdjust = true; if (pOneInterDataSrc->filePage.num <= pOneInterDataSrc->rowIdx) { - loadNewDataFromDiskFor(pLocalReducer, pOneInterDataSrc, &needToAdjust); + loadNewDataFromDiskFor(pLocalMerge, pOneInterDataSrc, &needToAdjust); } /* @@ -846,7 +844,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * * if the loser tree is rebuild completed, we do not need to adjust */ if (needToAdjust) { - int32_t leafNodeIdx = pTree->pNode[0].index + pLocalReducer->numOfBuffer; + int32_t leafNodeIdx = pTree->pNode[0].index + pLocalMerge->numOfBuffer; #ifdef _DEBUG_VIEW printf("before adjust:\t"); @@ -863,7 +861,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * } } -void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo, SFillInfo *pFillInfo) { +void savePrevRecordAndSetupFillInfo(SLocalMerger *pLocalMerge, SQueryInfo *pQueryInfo, SFillInfo *pFillInfo) { // discard following dataset in the same group and reset the interpolation information STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -876,28 +874,28 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ taosResetFillInfo(pFillInfo, revisedSTime); } - pLocalReducer->discard = true; - pLocalReducer->discardData->num = 0; + pLocalMerge->discard = true; + pLocalMerge->discardData->num = 0; - SColumnModel *pModel = pLocalReducer->pDesc->pColumnModel; - tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); + SColumnModel *pModel = pLocalMerge->pDesc->pColumnModel; + tColModelAppend(pModel, pLocalMerge->discardData, pLocalMerge->prevRowOfInput, 0, 1, 1); } -static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { +static void genFinalResWithoutFill(SSqlRes* pRes, SLocalMerger *pLocalMerge, SQueryInfo* pQueryInfo) { assert(pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); - tFilePage * pBeforeFillData = pLocalReducer->pResultBuf; + tFilePage * pBeforeFillData = pLocalMerge->pResultBuf; - pRes->data = pLocalReducer->pFinalRes; + pRes->data = pLocalMerge->pFinalRes; pRes->numOfRows = (int32_t) pBeforeFillData->num; if (pQueryInfo->limit.offset > 0) { if (pQueryInfo->limit.offset < pRes->numOfRows) { int32_t prevSize = (int32_t) pBeforeFillData->num; - tColModelErase(pLocalReducer->finalModel, pBeforeFillData, prevSize, 0, (int32_t)pQueryInfo->limit.offset - 1); + tColModelErase(pLocalMerge->finalModel, pBeforeFillData, prevSize, 0, (int32_t)pQueryInfo->limit.offset - 1); /* remove the hole in column model */ - tColModelCompact(pLocalReducer->finalModel, pBeforeFillData, prevSize); + tColModelCompact(pLocalMerge->finalModel, pBeforeFillData, prevSize); pRes->numOfRows -= (int32_t) pQueryInfo->limit.offset; pQueryInfo->limit.offset = 0; @@ -907,6 +905,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, } } + if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) { + pRes->numOfRows = 0; + pBeforeFillData->num = 0; + pLocalMerge->discard = true; + return; + } + pRes->numOfRowsGroup += pRes->numOfRows; // impose the limitation of output rows on the final result @@ -919,29 +924,29 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, pRes->numOfRows -= overflow; pBeforeFillData->num -= overflow; - tColModelCompact(pLocalReducer->finalModel, pBeforeFillData, prevSize); + tColModelCompact(pLocalMerge->finalModel, pBeforeFillData, prevSize); // set remain data to be discarded, and reset the interpolation information - savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo); + savePrevRecordAndSetupFillInfo(pLocalMerge, pQueryInfo, pLocalMerge->pFillInfo); } - memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalModel->rowSize)); + memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalMerge->finalModel->rowSize)); pRes->numOfClauseTotal += pRes->numOfRows; pBeforeFillData->num = 0; } /* - * Note: pRes->pLocalReducer may be null, due to the fact that "tscDestroyLocalReducer" is called + * Note: pRes->pLocalMerge may be null, due to the fact that "tscDestroyLocalMerger" is called * by "interuptHandler" function in shell */ -static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) { +static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutput) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - tFilePage *pBeforeFillData = pLocalReducer->pResultBuf; + tFilePage *pBeforeFillData = pLocalMerge->pResultBuf; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SFillInfo *pFillInfo = pLocalReducer->pFillInfo; + SFillInfo *pFillInfo = pLocalMerge->pFillInfo; // todo extract function int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; @@ -949,11 +954,11 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity); + pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalMerge->resColModel->capacity); } while (1) { - int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity); + int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalMerge->resColModel->capacity); if (pQueryInfo->limit.offset < newRows) { newRows -= pQueryInfo->limit.offset; @@ -966,7 +971,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } } - pRes->data = pLocalReducer->pFinalRes; + pRes->data = pLocalMerge->pFinalRes; pRes->numOfRows = (int32_t) newRows; pQueryInfo->limit.offset = 0; @@ -975,14 +980,13 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO pQueryInfo->limit.offset -= newRows; pRes->numOfRows = 0; - int32_t rpoints = taosNumOfRemainRows(pFillInfo); - if (rpoints <= 0) { + if (!taosFillHasMoreResults(pFillInfo)) { if (!doneOutput) { // reduce procedure has not completed yet, but current results for fill are exhausted break; } // all output in current group are completed - int32_t totalRemainRows = (int32_t)getNumOfResWithFill(pFillInfo, actualETime, pLocalReducer->resColModel->capacity); + int32_t totalRemainRows = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, actualETime, pLocalMerge->resColModel->capacity); if (totalRemainRows <= 0) { break; } @@ -1000,7 +1004,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO assert(pRes->numOfRows >= 0); /* set remain data to be discarded, and reset the interpolation information */ - savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo); + savePrevRecordAndSetupFillInfo(pLocalMerge, pQueryInfo, pFillInfo); } int32_t offset = 0; @@ -1022,8 +1026,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO tfree(pResPages); } -static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { - SColumnModel *pColumnModel = pLocalReducer->pDesc->pColumnModel; +static void savePreviousRow(SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) { + SColumnModel *pColumnModel = pLocalMerge->pDesc->pColumnModel; assert(pColumnModel->capacity == 1 && tmpBuffer->num == 1); // copy to previous temp buffer @@ -1031,26 +1035,26 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) SSchema *pSchema = getColumnModelSchema(pColumnModel, i); int16_t offset = getColumnModelOffset(pColumnModel, i); - memcpy(pLocalReducer->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes); + memcpy(pLocalMerge->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes); } tmpBuffer->num = 0; - pLocalReducer->hasPrevRow = true; + pLocalMerge->hasPrevRow = true; } -static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) { +static void doExecuteFinalMerge(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, bool needInit) { // the tag columns need to be set before all functions execution SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t j = 0; j < size; ++j) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[j]; // tags/tags_dummy function, the tag field of SQLFunctionCtx is from the input buffer int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS_DUMMY) { tVariantDestroy(&pCtx->tag); - char* input = pCtx->aInputElemBuf; + char* input = pCtx->pInput; if (pCtx->inputType == TSDB_DATA_TYPE_BINARY || pCtx->inputType == TSDB_DATA_TYPE_NCHAR) { assert(varDataLen(input) <= pCtx->inputBytes); @@ -1058,12 +1062,13 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } else { tVariantCreateFromBinary(&pCtx->tag, input, pCtx->inputBytes, pCtx->inputType); } + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, j); - pCtx->param[0].i64Key = pExpr->param[0].i64Key; + pCtx->param[0].i64 = pExpr->param[0].i64; } - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; if (needInit) { aAggs[pCtx->functionId].init(pCtx); @@ -1071,20 +1076,20 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } for (int32_t j = 0; j < size; ++j) { - int32_t functionId = pLocalReducer->pCtx[j].functionId; + int32_t functionId = pLocalMerge->pCtx[j].functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { continue; } - aAggs[functionId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); + aAggs[functionId].mergeFunc(&pLocalMerge->pCtx[j]); } } -static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { - if (pLocalReducer->hasUnprocessedRow) { - pLocalReducer->hasUnprocessedRow = false; - doExecuteSecondaryMerge(pCmd, pLocalReducer, true); - savePreviousRow(pLocalReducer, tmpBuffer); +static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) { + if (pLocalMerge->hasUnprocessedRow) { + pLocalMerge->hasUnprocessedRow = false; + doExecuteFinalMerge(pCmd, pLocalMerge, true); + savePreviousRow(pLocalMerge, tmpBuffer); } } @@ -1098,7 +1103,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) * the number of output result is decided by main output */ int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) { continue; } @@ -1117,7 +1122,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) * filled with the same result, which is the tags, specified in group by clause * */ -static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalReducer *pLocalReducer) { +static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalMerger *pLocalMerge) { int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer size_t size = tscSqlExprNumOfExprs(pQueryInfo); @@ -1132,38 +1137,38 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo char *buf = malloc((size_t)maxBufSize); for (int32_t k = 0; k < size; ++k) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[k]; if (pCtx->functionId != TSDB_FUNC_TAG) { continue; } int32_t inc = numOfRes - 1; // tsdb_func_tag function only produce one row of result memset(buf, 0, (size_t)maxBufSize); - memcpy(buf, pCtx->aOutputBuf, (size_t)pCtx->outputBytes); + memcpy(buf, pCtx->pOutput, (size_t)pCtx->outputBytes); for (int32_t i = 0; i < inc; ++i) { - pCtx->aOutputBuf += pCtx->outputBytes; - memcpy(pCtx->aOutputBuf, buf, (size_t)pCtx->outputBytes); + pCtx->pOutput += pCtx->outputBytes; + memcpy(pCtx->pOutput, buf, (size_t)pCtx->outputBytes); } } free(buf); } -int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { +int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge) { size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t k = 0; k < size; ++k) { - SQLFunctionCtx* pCtx = &pLocalReducer->pCtx[k]; + SQLFunctionCtx* pCtx = &pLocalMerge->pCtx[k]; aAggs[pCtx->functionId].xFinalize(pCtx); } - pLocalReducer->hasPrevRow = false; + pLocalMerge->hasPrevRow = false; - int32_t numOfRes = (int32_t)getNumOfResultLocal(pQueryInfo, pLocalReducer->pCtx); - pLocalReducer->pResultBuf->num += numOfRes; + int32_t numOfRes = (int32_t)getNumOfResultLocal(pQueryInfo, pLocalMerge->pCtx); + pLocalMerge->pResultBuf->num += numOfRes; - fillMultiRowsOfTagsVal(pQueryInfo, numOfRes, pLocalReducer); + fillMultiRowsOfTagsVal(pQueryInfo, numOfRes, pLocalMerge); return numOfRes; } @@ -1174,22 +1179,22 @@ int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { * results generated by simple aggregation function, we merge them all into one points * *Exception*: column projection query, required no merge procedure */ -bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { +bool needToMerge(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) { int32_t ret = 0; // merge all result by default - int16_t functionId = pLocalReducer->pCtx[0].functionId; + int16_t functionId = pLocalMerge->pCtx[0].functionId; // todo opt performance - if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0))) { // column projection query + if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0) && pQueryInfo->distinctTag == false)) { // column projection query ret = 1; // disable merge procedure } else { - tOrderDescriptor *pDesc = pLocalReducer->pDesc; + tOrderDescriptor *pDesc = pLocalMerge->pDesc; if (pDesc->orderInfo.numOfCols > 0) { if (pDesc->tsOrder == TSDB_ORDER_ASC) { // asc // todo refactor comparator - ret = compare_a(pLocalReducer->pDesc, 1, 0, pLocalReducer->prevRowOfInput, 1, 0, tmpBuffer->data); + ret = compare_a(pLocalMerge->pDesc, 1, 0, pLocalMerge->prevRowOfInput, 1, 0, tmpBuffer->data); } else { // desc - ret = compare_d(pLocalReducer->pDesc, 1, 0, pLocalReducer->prevRowOfInput, 1, 0, tmpBuffer->data); + ret = compare_d(pLocalMerge->pDesc, 1, 0, pLocalMerge->prevRowOfInput, 1, 0, tmpBuffer->data); } } } @@ -1227,17 +1232,17 @@ static bool saveGroupResultInfo(SSqlObj *pSql) { /** * * @param pSql - * @param pLocalReducer + * @param pLocalMerge * @param noMoreCurrentGroupRes * @return if current group is skipped, return false, and do NOT record it into pRes->numOfGroups */ -bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCurrentGroupRes) { +bool genFinalResults(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool noMoreCurrentGroupRes) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - tFilePage * pResBuf = pLocalReducer->pResultBuf; - SColumnModel *pModel = pLocalReducer->resColModel; + tFilePage * pResBuf = pLocalMerge->pResultBuf; + SColumnModel *pModel = pLocalMerge->resColModel; pRes->code = TSDB_CODE_SUCCESS; @@ -1248,11 +1253,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur if (pQueryInfo->slimit.offset > 0) { pRes->numOfRows = 0; pQueryInfo->slimit.offset -= 1; - pLocalReducer->discard = !noMoreCurrentGroupRes; + pLocalMerge->discard = !noMoreCurrentGroupRes; - if (pLocalReducer->discard) { - SColumnModel *pInternModel = pLocalReducer->pDesc->pColumnModel; - tColModelAppend(pInternModel, pLocalReducer->discardData, pLocalReducer->pTempBuffer->data, 0, 1, 1); + if (pLocalMerge->discard) { + SColumnModel *pInternModel = pLocalMerge->pDesc->pColumnModel; + tColModelAppend(pInternModel, pLocalMerge->discardData, pLocalMerge->pTempBuffer->data, 0, 1, 1); } return false; @@ -1261,19 +1266,14 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur tColModelCompact(pModel, pResBuf, pModel->capacity); if (tscIsSecondStageQuery(pQueryInfo)) { - doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalModel->rowSize); + doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalMerge->finalModel->rowSize); } -#ifdef _DEBUG_VIEW - printf("final result before interpo:\n"); -// tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num); -#endif - // no interval query, no fill operation if (pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { - genFinalResWithoutFill(pRes, pLocalReducer, pQueryInfo); + genFinalResWithoutFill(pRes, pLocalMerge, pQueryInfo); } else { - SFillInfo* pFillInfo = pLocalReducer->pFillInfo; + SFillInfo* pFillInfo = pLocalMerge->pFillInfo; if (pFillInfo != NULL) { TSKEY ekey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; @@ -1281,30 +1281,34 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } - doFillResult(pSql, pLocalReducer, noMoreCurrentGroupRes); + doFillResult(pSql, pLocalMerge, noMoreCurrentGroupRes); } return true; } -void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning +void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge) {// reset output buffer to the beginning size_t t = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < t; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity; + pLocalMerge->pCtx[i].pOutput = pLocalMerge->pResultBuf->data + pExpr->offset * pLocalMerge->resColModel->capacity; + + if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM || pExpr->functionId == TSDB_FUNC_DIFF) { + pLocalMerge->pCtx[i].ptsOutputBuf = pLocalMerge->pCtx[0].pOutput; + } } - memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage)); + memset(pLocalMerge->pResultBuf, 0, pLocalMerge->nResultBufSize + sizeof(tFilePage)); } -static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { +static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalMerger *pLocalMerge) { // In handling data in other groups, we need to reset the interpolation information for a new group data pRes->numOfRows = 0; pRes->numOfRowsGroup = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pQueryInfo->limit.offset = pLocalReducer->offset; + pQueryInfo->limit.offset = pLocalMerge->offset; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -1313,12 +1317,12 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer if (pQueryInfo->fillType != TSDB_FILL_NONE) { TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t newTime = taosTimeTruncate(skey, &pQueryInfo->interval, tinfo.precision); - taosResetFillInfo(pLocalReducer->pFillInfo, newTime); + taosResetFillInfo(pLocalMerge->pFillInfo, newTime); } } -static bool isAllSourcesCompleted(SLocalReducer *pLocalReducer) { - return (pLocalReducer->numOfBuffer == pLocalReducer->numOfCompleted); +static bool isAllSourcesCompleted(SLocalMerger *pLocalMerge) { + return (pLocalMerge->numOfBuffer == pLocalMerge->numOfCompleted); } static bool doBuildFilledResultForGroup(SSqlObj *pSql) { @@ -1326,19 +1330,19 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SFillInfo *pFillInfo = pLocalReducer->pFillInfo; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; + SFillInfo *pFillInfo = pLocalMerge->pFillInfo; - if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) { + if (pFillInfo != NULL && taosFillHasMoreResults(pFillInfo)) { assert(pQueryInfo->fillType != TSDB_FILL_NONE); - tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf; + tFilePage *pFinalDataBuf = pLocalMerge->pResultBuf; int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1)); // the first column must be the timestamp column - int32_t rows = (int32_t) getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity); + int32_t rows = (int32_t) getNumOfResultsAfterFillGap(pFillInfo, etime, pLocalMerge->resColModel->capacity); if (rows > 0) { // do fill gap - doFillResult(pSql, pLocalReducer, false); + doFillResult(pSql, pLocalMerge, false); } return true; @@ -1351,23 +1355,23 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SFillInfo *pFillInfo = pLocalReducer->pFillInfo; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; + SFillInfo *pFillInfo = pLocalMerge->pFillInfo; - bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow; + bool prevGroupCompleted = (!pLocalMerge->discard) && pLocalMerge->hasUnprocessedRow; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || + if ((isAllSourcesCompleted(pLocalMerge) && !pLocalMerge->hasPrevRow) || pLocalMerge->pLocalDataSrc[0] == NULL || prevGroupCompleted) { // if fillType == TSDB_FILL_NONE, return directly if (pQueryInfo->fillType != TSDB_FILL_NONE && ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey; - int32_t rows = (int32_t)getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity); + int32_t rows = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, etime, pLocalMerge->resColModel->capacity); if (rows > 0) { - doFillResult(pSql, pLocalReducer, true); + doFillResult(pSql, pLocalMerge, true); } } @@ -1377,7 +1381,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { * * No results will be generated and query completed. */ - if (pRes->numOfRows > 0 || (isAllSourcesCompleted(pLocalReducer) && (!pLocalReducer->hasUnprocessedRow))) { + if (pRes->numOfRows > 0 || (isAllSourcesCompleted(pLocalMerge) && (!pLocalMerge->hasUnprocessedRow))) { return true; } @@ -1386,7 +1390,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { return true; } - resetEnvForNewResultset(pRes, pCmd, pLocalReducer); + resetEnvForNewResultset(pRes, pCmd, pLocalMerge); } return false; @@ -1396,13 +1400,13 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - SLocalReducer *pLocalReducer = pRes->pLocalReducer; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t k = 0; k < size; ++k) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; - pCtx->aOutputBuf += pCtx->outputBytes * numOfRes; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[k]; + pCtx->pOutput += pCtx->outputBytes * numOfRes; // set the correct output timestamp column position if (pCtx->functionId == TSDB_FUNC_TOP || pCtx->functionId == TSDB_FUNC_BOTTOM) { @@ -1410,7 +1414,7 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { } } - doExecuteSecondaryMerge(pCmd, pLocalReducer, true); + doExecuteFinalMerge(pCmd, pLocalMerge, true); } int32_t tscDoLocalMerge(SSqlObj *pSql) { @@ -1419,52 +1423,45 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tscResetForNextRetrieve(pRes); - if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed + if (pSql->signature != pSql || pRes == NULL || pRes->pLocalMerger == NULL) { // all data has been processed + if (pRes->code == TSDB_CODE_SUCCESS) { + return pRes->code; + } + tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); return pRes->code; } - SLocalReducer *pLocalReducer = pRes->pLocalReducer; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - // set the data merge in progress - int32_t prevStatus = - atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); - if (prevStatus != TSC_LOCALREDUCE_READY) { - assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already - return TSDB_CODE_SUCCESS; - } - - tFilePage *tmpBuffer = pLocalReducer->pTempBuffer; + tFilePage *tmpBuffer = pLocalMerge->pTempBuffer; if (doHandleLastRemainData(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } if (doBuildFilledResultForGroup(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } - SLoserTreeInfo *pTree = pLocalReducer->pLoserTree; + SLoserTreeInfo *pTree = pLocalMerge->pLoserTree; // clear buffer - handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); - SColumnModel *pModel = pLocalReducer->pDesc->pColumnModel; + handleUnprocessedRow(pCmd, pLocalMerge, tmpBuffer); + SColumnModel *pModel = pLocalMerge->pDesc->pColumnModel; while (1) { - if (isAllSourcesCompleted(pLocalReducer)) { + if (isAllSourcesCompleted(pLocalMerge)) { break; } #ifdef _DEBUG_VIEW printf("chosen data in pTree[0] = %d\n", pTree->pNode[0].index); #endif - assert((pTree->pNode[0].index < pLocalReducer->numOfBuffer) && (pTree->pNode[0].index >= 0) && tmpBuffer->num == 0); + assert((pTree->pNode[0].index < pLocalMerge->numOfBuffer) && (pTree->pNode[0].index >= 0) && tmpBuffer->num == 0); // chosen from loser tree - SLocalDataSource *pOneDataSrc = pLocalReducer->pLocalDataSrc[pTree->pNode[0].index]; + SLocalDataSource *pOneDataSrc = pLocalMerge->pLocalDataSrc[pTree->pNode[0].index]; tColModelAppend(pModel, tmpBuffer, pOneDataSrc->filePage.data, pOneDataSrc->rowIdx, 1, pOneDataSrc->pMemBuffer->pColumnModel->capacity); @@ -1477,78 +1474,76 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->num, pModel->capacity, colInfo); #endif - if (pLocalReducer->discard) { - assert(pLocalReducer->hasUnprocessedRow == false); + if (pLocalMerge->discard) { + assert(pLocalMerge->hasUnprocessedRow == false); /* current record belongs to the same group of previous record, need to discard it */ - if (isSameGroup(pCmd, pLocalReducer, pLocalReducer->discardData->data, tmpBuffer)) { + if (isSameGroup(pCmd, pLocalMerge, pLocalMerge->discardData->data, tmpBuffer)) { tmpBuffer->num = 0; pOneDataSrc->rowIdx += 1; - adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); + adjustLoserTreeFromNewData(pLocalMerge, pOneDataSrc, pTree); // all inputs are exhausted, abort current process - if (isAllSourcesCompleted(pLocalReducer)) { + if (isAllSourcesCompleted(pLocalMerge)) { break; } // data belongs to the same group needs to be discarded continue; } else { - pLocalReducer->discard = false; - pLocalReducer->discardData->num = 0; + pLocalMerge->discard = false; + pLocalMerge->discardData->num = 0; if (saveGroupResultInfo(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; return TSDB_CODE_SUCCESS; } - resetEnvForNewResultset(pRes, pCmd, pLocalReducer); + resetEnvForNewResultset(pRes, pCmd, pLocalMerge); } } - if (pLocalReducer->hasPrevRow) { - if (needToMerge(pQueryInfo, pLocalReducer, tmpBuffer)) { + if (pLocalMerge->hasPrevRow) { + if (needToMerge(pQueryInfo, pLocalMerge, tmpBuffer)) { // belong to the group of the previous row, continue process it - doExecuteSecondaryMerge(pCmd, pLocalReducer, false); + doExecuteFinalMerge(pCmd, pLocalMerge, false); // copy to buffer - savePreviousRow(pLocalReducer, tmpBuffer); + savePreviousRow(pLocalMerge, tmpBuffer); } else { /* * current row does not belong to the group of previous row. * so the processing of previous group is completed. */ - int32_t numOfRes = finalizeRes(pQueryInfo, pLocalReducer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); + int32_t numOfRes = finalizeRes(pQueryInfo, pLocalMerge); + bool sameGroup = isSameGroup(pCmd, pLocalMerge, pLocalMerge->prevRowOfInput, tmpBuffer); - tFilePage *pResBuf = pLocalReducer->pResultBuf; + tFilePage *pResBuf = pLocalMerge->pResultBuf; /* * if the previous group does NOT generate any result (pResBuf->num == 0), * continue to process results instead of return results. */ - if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num == pLocalReducer->resColModel->capacity)) { + if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num == pLocalMerge->resColModel->capacity)) { // does not belong to the same group - bool notSkipped = genFinalResults(pSql, pLocalReducer, !sameGroup); + bool notSkipped = genFinalResults(pSql, pLocalMerge, !sameGroup); // this row needs to discard, since it belongs to the group of previous - if (pLocalReducer->discard && sameGroup) { - pLocalReducer->hasUnprocessedRow = false; + if (pLocalMerge->discard && sameGroup) { + pLocalMerge->hasUnprocessedRow = false; tmpBuffer->num = 0; } else { // current row does not belongs to the previous group, so it is not be handled yet. - pLocalReducer->hasUnprocessedRow = true; + pLocalMerge->hasUnprocessedRow = true; } - resetOutputBuf(pQueryInfo, pLocalReducer); + resetOutputBuf(pQueryInfo, pLocalMerge); pOneDataSrc->rowIdx += 1; // here we do not check the return value - adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); - assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS); + adjustLoserTreeFromNewData(pLocalMerge, pOneDataSrc, pTree); if (pRes->numOfRows == 0) { - handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); + handleUnprocessedRow(pCmd, pLocalMerge, tmpBuffer); if (!sameGroup) { /* @@ -1556,11 +1551,10 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * If previous group is not skipped, keep it in pRes->numOfGroups */ if (notSkipped && saveGroupResultInfo(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; return TSDB_CODE_SUCCESS; } - resetEnvForNewResultset(pRes, pCmd, pLocalReducer); + resetEnvForNewResultset(pRes, pCmd, pLocalMerge); } } else { /* @@ -1568,7 +1562,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * We start the process in a new round. */ if (sameGroup) { - handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); + handleUnprocessedRow(pCmd, pLocalMerge, tmpBuffer); } } @@ -1576,41 +1570,37 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { if (pRes->numOfRows == 0) { continue; } else { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } } else { // result buffer is not full doProcessResultInNextWindow(pSql, numOfRes); - savePreviousRow(pLocalReducer, tmpBuffer); + savePreviousRow(pLocalMerge, tmpBuffer); } } } else { - doExecuteSecondaryMerge(pCmd, pLocalReducer, true); - savePreviousRow(pLocalReducer, tmpBuffer); // copy the processed row to buffer + doExecuteFinalMerge(pCmd, pLocalMerge, true); + savePreviousRow(pLocalMerge, tmpBuffer); // copy the processed row to buffer } pOneDataSrc->rowIdx += 1; - adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); + adjustLoserTreeFromNewData(pLocalMerge, pOneDataSrc, pTree); } - if (pLocalReducer->hasPrevRow) { - finalizeRes(pQueryInfo, pLocalReducer); + if (pLocalMerge->hasPrevRow) { + finalizeRes(pQueryInfo, pLocalMerge); } - if (pLocalReducer->pResultBuf->num) { - genFinalResults(pSql, pLocalReducer, true); + if (pLocalMerge->pResultBuf->num) { + genFinalResults(pSql, pLocalMerge, true); } - assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS && pRes->row == 0); - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. - return TSDB_CODE_SUCCESS; } void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) { SSqlRes *pRes = &pObj->res; - if (pRes->pLocalReducer != NULL) { - tscDestroyLocalReducer(pObj); + if (pRes->pLocalMerger != NULL) { + tscDestroyLocalMerger(pObj); } pRes->qhandle = 1; // hack to pass the safety check in fetch_row function @@ -1618,22 +1608,22 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) pRes->row = 0; pRes->rspType = 0; // used as a flag to denote if taos_retrieved() has been called yet - pRes->pLocalReducer = (SLocalReducer *)calloc(1, sizeof(SLocalReducer)); + pRes->pLocalMerger = (SLocalMerger *)calloc(1, sizeof(SLocalMerger)); /* * we need one additional byte space * the sprintf function needs one additional space to put '\0' at the end of string */ size_t allocSize = numOfRes * rowLen + sizeof(tFilePage) + 1; - pRes->pLocalReducer->pResultBuf = (tFilePage *)calloc(1, allocSize); + pRes->pLocalMerger->pResultBuf = (tFilePage *)calloc(1, allocSize); - pRes->pLocalReducer->pResultBuf->num = numOfRes; - pRes->data = pRes->pLocalReducer->pResultBuf->data; + pRes->pLocalMerger->pResultBuf->num = numOfRes; + pRes->data = pRes->pLocalMerger->pResultBuf->data; } int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { int32_t maxRowSize = MAX(rowSize, finalRowSize); - char* pbuf = calloc(1, pOutput->num * maxRowSize); + char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); size_t size = tscNumOfFields(pQueryInfo); SArithmeticSupport arithSup = {0}; @@ -1657,19 +1647,19 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ // calculate the result from several other columns if (pSup->pArithExprInfo != NULL) { arithSup.pArithExpr = pSup->pArithExprInfo; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); } else { SSqlExpr* pExpr = pSup->pSqlExpr; - memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, pExpr->resBytes * pOutput->num); + memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num)); } offset += pSup->field.bytes; } - memcpy(pOutput->data, pbuf, pOutput->num * offset); + memcpy(pOutput->data, pbuf, (size_t)(pOutput->num * offset)); tfree(pbuf); tfree(arithSup.data); return offset; -} \ No newline at end of file +} diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 18e5b6f0744a06e5d28f2ce5cfa3df1244a0dc89..a5906f5539a8d40b055f580933feb4b1184331f1 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -20,6 +20,7 @@ #include "os.h" +#include "ttype.h" #include "hash.h" #include "tscUtil.h" #include "tschemautil.h" @@ -40,46 +41,9 @@ enum { static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); -static int32_t tscToInteger(SStrToken *pToken, int64_t *value, char **endPtr) { - if (pToken->n == 0) { - return TK_ILLEGAL; - } - - - int32_t radix = 10; - if (pToken->type == TK_HEX) { - radix = 16; - } else if (pToken->type == TK_BIN) { - radix = 2; - } - - errno = 0; - *value = strtoll(pToken->z, endPtr, radix); - if (**endPtr == 'e' || **endPtr == 'E' || **endPtr == '.') { - errno = 0; - double v = round(strtod(pToken->z, endPtr)); - if (v > INT64_MAX || v <= INT64_MIN) { - errno = ERANGE; - } else { - *value = (int64_t)v; - } - } - - // not a valid integer number, return error - if (*endPtr - pToken->z != pToken->n) { - return TK_ILLEGAL; - } - - return pToken->type; -} - static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { - if (pToken->n == 0) { - return TK_ILLEGAL; - } - errno = 0; - *value = strtod(pToken->z, endPtr); + *value = strtold(pToken->z, endPtr); // not a valid integer number, return error if ((*endPtr - pToken->z) != pToken->n) { @@ -163,86 +127,119 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } +// todo extract the null value check +static bool isNullStr(SStrToken* pToken) { + return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); +} int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec) { int64_t iv; - int32_t numType; - char * endptr = NULL; - errno = 0; // clear the previous existed error information + int32_t ret; + char *endptr = NULL; + + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return tscInvalidSQLErrMsg(msg, "invalid numeric data", pToken->z); + } switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { // bool - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_TRUE; - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_FALSE; - } else if (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0) { - *(uint8_t *)payload = TSDB_DATA_BOOL_NULL; + if (isNullStr(pToken)) { + *((uint8_t *)payload) = TSDB_DATA_BOOL_NULL; + } else { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + *(uint8_t *)payload = TSDB_TRUE; + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + *(uint8_t *)payload = TSDB_FALSE; + } else { + return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_INTEGER) { + iv = strtoll(pToken->z, NULL, 10); + *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); + } else if (pToken->type == TK_FLOAT) { + double dv = strtod(pToken->z, NULL); + *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); } else { - return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); + return tscInvalidSQLErrMsg(msg, "invalid bool data", pToken->z); } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else if (pToken->type == TK_NULL) { - *(uint8_t *)payload = TSDB_DATA_BOOL_NULL; - } else { - return tscInvalidSQLErrMsg(msg, "invalid bool data", pToken->z); } break; } + case TSDB_DATA_TYPE_TINYINT: - if (pToken->type == TK_NULL) { - *((int8_t *)payload) = TSDB_DATA_TINYINT_NULL; - } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { - *((int8_t *)payload) = TSDB_DATA_TINYINT_NULL; + if (isNullStr(pToken)) { + *((uint8_t *)payload) = TSDB_DATA_TINYINT_NULL; } else { - numType = tscToInteger(pToken, &iv, &endptr); - if (TK_ILLEGAL == numType) { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { return tscInvalidSQLErrMsg(msg, "invalid tinyint data", pToken->z); - } else if (errno == ERANGE || iv > INT8_MAX || iv <= INT8_MIN) { - return tscInvalidSQLErrMsg(msg, "tinyint data overflow", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return tscInvalidSQLErrMsg(msg, "data overflow", pToken->z); } - *((int8_t *)payload) = (int8_t)iv; + *((uint8_t *)payload) = (uint8_t)iv; + } + + break; + + case TSDB_DATA_TYPE_UTINYINT: + if (isNullStr(pToken)) { + *((uint8_t *)payload) = TSDB_DATA_UTINYINT_NULL; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidSQLErrMsg(msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return tscInvalidSQLErrMsg(msg, "unsigned tinyint data overflow", pToken->z); + } + + *((uint8_t *)payload) = (uint8_t)iv; } break; case TSDB_DATA_TYPE_SMALLINT: - if (pToken->type == TK_NULL) { - *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; - } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + if (isNullStr(pToken)) { *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; } else { - numType = tscToInteger(pToken, &iv, &endptr); - if (TK_ILLEGAL == numType) { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { return tscInvalidSQLErrMsg(msg, "invalid smallint data", pToken->z); - } else if (errno == ERANGE || iv > INT16_MAX || iv <= INT16_MIN) { + } else if (!IS_VALID_SMALLINT(iv)) { return tscInvalidSQLErrMsg(msg, "smallint data overflow", pToken->z); } *((int16_t *)payload) = (int16_t)iv; } + + break; + + case TSDB_DATA_TYPE_USMALLINT: + if (isNullStr(pToken)) { + *((uint16_t *)payload) = TSDB_DATA_USMALLINT_NULL; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidSQLErrMsg(msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return tscInvalidSQLErrMsg(msg, "unsigned smallint data overflow", pToken->z); + } + + *((uint16_t *)payload) = (uint16_t)iv; + } + break; case TSDB_DATA_TYPE_INT: - if (pToken->type == TK_NULL) { - *((int32_t *)payload) = TSDB_DATA_INT_NULL; - } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + if (isNullStr(pToken)) { *((int32_t *)payload) = TSDB_DATA_INT_NULL; } else { - numType = tscToInteger(pToken, &iv, &endptr); - if (TK_ILLEGAL == numType) { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { return tscInvalidSQLErrMsg(msg, "invalid int data", pToken->z); - } else if (errno == ERANGE || iv > INT32_MAX || iv <= INT32_MIN) { + } else if (!IS_VALID_INT(iv)) { return tscInvalidSQLErrMsg(msg, "int data overflow", pToken->z); } @@ -251,17 +248,30 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, break; + case TSDB_DATA_TYPE_UINT: + if (isNullStr(pToken)) { + *((uint32_t *)payload) = TSDB_DATA_UINT_NULL; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidSQLErrMsg(msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return tscInvalidSQLErrMsg(msg, "unsigned int data overflow", pToken->z); + } + + *((uint32_t *)payload) = (uint32_t)iv; + } + + break; + case TSDB_DATA_TYPE_BIGINT: - if (pToken->type == TK_NULL) { - *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; - } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + if (isNullStr(pToken)) { *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; } else { - numType = tscToInteger(pToken, &iv, &endptr); - if (TK_ILLEGAL == numType) { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { return tscInvalidSQLErrMsg(msg, "invalid bigint data", pToken->z); - } else if (errno == ERANGE || iv == INT64_MIN) { + } else if (!IS_VALID_BIGINT(iv)) { return tscInvalidSQLErrMsg(msg, "bigint data overflow", pToken->z); } @@ -269,11 +279,23 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, } break; + case TSDB_DATA_TYPE_UBIGINT: + if (isNullStr(pToken)) { + *((uint64_t *)payload) = TSDB_DATA_UBIGINT_NULL; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidSQLErrMsg(msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return tscInvalidSQLErrMsg(msg, "unsigned bigint data overflow", pToken->z); + } + + *((uint64_t *)payload) = iv; + } + break; + case TSDB_DATA_TYPE_FLOAT: - if (pToken->type == TK_NULL) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; - } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + if (isNullStr(pToken)) { *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; } else { double dv; @@ -281,24 +303,16 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, return tscInvalidSQLErrMsg(msg, "illegal float data", pToken->z); } - float fv = (float)dv; - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || (fv > FLT_MAX || fv < -FLT_MAX)) { + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return tscInvalidSQLErrMsg(msg, "illegal float data", pToken->z); } - if (isinf(fv) || isnan(fv)) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; - } - - *((float *)payload) = fv; + *((float *)payload) = (float)dv; } break; case TSDB_DATA_TYPE_DOUBLE: - if (pToken->type == TK_NULL) { - *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; - } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + if (isNullStr(pToken)) { *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; } else { double dv; @@ -306,15 +320,11 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || (dv > DBL_MAX || dv < -DBL_MAX)) { + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z); } - if (isinf(dv) || isnan(dv)) { - *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; - } else { - *((double *)payload) = dv; - } + *((double *)payload) = dv; } break; @@ -337,7 +347,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - size_t output = 0; + int32_t output = 0; if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { char buf[512] = {0}; snprintf(buf, tListLen(buf), "%s", strerror(errno)); @@ -686,17 +696,14 @@ void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { } } -static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **str, SParsedDataColInfo *spd, - int32_t *totalNum) { - SSqlCmd * pCmd = &pSql->cmd; +static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColInfo *spd, int32_t *totalNum) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; STableComInfo tinfo = tscGetTableInfo(pTableMeta); STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, - pTableMeta, &dataBuf); + int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &dataBuf, NULL); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -734,7 +741,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st return code; } - dataBuf->vgId = pTableMeta->vgroupInfo.vgId; + dataBuf->vgId = pTableMeta->vgId; dataBuf->numOfTables = 1; *totalNum += numOfRows; @@ -755,6 +762,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { char *sql = *sqlstr; + pSql->cmd.autoCreated = false; + // get the token of specified table index = 0; tableToken = tStrGetToken(sql, &index, false, 0, NULL); @@ -799,38 +808,31 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sToken = tStrGetToken(sql, &index, false, 0, NULL); sql += index; - tscAllocPayload(pCmd, sizeof(STagData)); - //the source super table is moved to the secondary position of the pTableMetaInfo list if (pQueryInfo->numOfTables < 2) { tscAddEmptyMetaInfo(pQueryInfo); } - STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - code = tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql); + STableMetaInfo *pSTableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); + code = tscSetTableFullName(pSTableMetaInfo, &sToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } - STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data)); - if (pTag == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memset(pTag, 0, offsetof(STagData, data)); - tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name)); - pCmd->pTagData = pTag; + tNameExtractFullName(&pSTableMetaInfo->name, pCmd->tagData.name); + pCmd->tagData.dataLen = 0; - code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); + code = tscGetTableMeta(pSql, pSTableMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } - if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMeterMetaInfo)) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMetaInfo)) { return tscInvalidSQLErrMsg(pCmd->payload, "create table only from super table is allowed", sToken.z); } - SSchema *pTagSchema = tscGetTableTagSchema(pSTableMeterMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pSTableMeterMetaInfo->pTableMeta); + SSchema *pTagSchema = tscGetTableTagSchema(pSTableMetaInfo->pTableMeta); + STableComInfo tinfo = tscGetTableInfo(pSTableMetaInfo->pTableMeta); index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); @@ -838,7 +840,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { SParsedDataColInfo spd = {0}; - uint8_t numOfTags = tscGetNumOfTags(pSTableMeterMetaInfo->pTableMeta); + uint8_t numOfTags = tscGetNumOfTags(pSTableMetaInfo->pTableMeta); spd.numOfCols = numOfTags; // if specify some tags column @@ -903,6 +905,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z); } + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + if (sToken.type != TK_LP) { + return tscInvalidSQLErrMsg(pCmd->payload, NULL, sToken.z); + } + SKVRowBuilder kvRowBuilder = {0}; if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -945,18 +954,23 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder); if (row == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return tscInvalidSQLErrMsg(pCmd->payload, "tag value expected", NULL); } tdSortKVRowByColIdx(row); - pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row)); + pCmd->tagData.dataLen = kvRowLen(row); + if (pCmd->tagData.dataLen <= 0){ + return tscInvalidSQLErrMsg(pCmd->payload, "tag value expected", NULL); + } + + char* pTag = realloc(pCmd->tagData.data, pCmd->tagData.dataLen); if (pTag == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pCmd->pTagData = pTag; - pTag->dataLen = htonl(kvRowLen(row)); - kvRowCpy(pTag->data, row); + + kvRowCpy(pTag, row); free(row); + pCmd->tagData.data = pTag; index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); @@ -975,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } createTable = true; - code = tscGetMeterMetaEx(pSql, pTableMetaInfo, true); + code = tscGetTableMetaEx(pSql, pTableMetaInfo, true); if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { return code; } @@ -986,7 +1000,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } else { sql = sToken.z; } - code = tscGetMeterMetaEx(pSql, pTableMetaInfo, false); + code = tscGetTableMetaEx(pSql, pTableMetaInfo, false); if (pCmd->curSql == NULL) { assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS); @@ -1029,11 +1043,7 @@ static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) { } /** - * usage: insert into table1 values() () table2 values()() - * - * @param str - * @param acct - * @param db + * parse insert sql * @param pSql * @return */ @@ -1058,18 +1068,17 @@ int tsParseInsertSql(SSqlObj *pSql) { return code; } - if (NULL == pCmd->pTableList) { - pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - pCmd->pDataBlocks = taosArrayInit(4, POINTER_BYTES); - if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) { + if (NULL == pCmd->pTableBlockHashList) { + pCmd->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (NULL == pCmd->pTableBlockHashList) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; + goto _clean; } } else { str = pCmd->curSql; } - tscDebug("%p create data block list for submit data:%p, pTableList:%p", pSql, pCmd->pDataBlocks, pCmd->pTableList); + tscDebug("%p create data block list hashList:%p", pSql, pCmd->pTableBlockHashList); while (1) { int32_t index = 0; @@ -1091,7 +1100,7 @@ int tsParseInsertSql(SSqlObj *pSql) { */ if (totalNum == 0) { code = TSDB_CODE_TSC_INVALID_SQL; - goto _error; + goto _clean; } else { break; } @@ -1104,11 +1113,11 @@ int tsParseInsertSql(SSqlObj *pSql) { // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z); - goto _error; + goto _clean; } if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) { - goto _error; + goto _clean; } if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) { @@ -1122,12 +1131,12 @@ int tsParseInsertSql(SSqlObj *pSql) { tscError("%p async insert parse error, code:%s", pSql, tstrerror(code)); pCmd->curSql = NULL; - goto _error; + goto _clean; } if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL); - goto _error; + goto _clean; } index = 0; @@ -1136,7 +1145,7 @@ int tsParseInsertSql(SSqlObj *pSql) { if (sToken.n == 0) { code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE required", sToken.z); - goto _error; + goto _clean; } STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -1148,32 +1157,32 @@ int tsParseInsertSql(SSqlObj *pSql) { tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns); if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { - goto _error; + goto _clean; } /* * app here insert data in different vnodes, so we need to set the following * data in another submit procedure using async insert routines */ - code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum); + code = doParseInsertStatement(pCmd, &str, &spd, &totalNum); if (code != TSDB_CODE_SUCCESS) { - goto _error; + goto _clean; } } else if (sToken.type == TK_FILE) { if (validateDataSource(pCmd, DATA_FROM_DATA_FILE, sToken.z) != TSDB_CODE_SUCCESS) { - goto _error; + goto _clean; } index = 0; sToken = tStrGetToken(str, &index, false, 0, NULL); if (sToken.type != TK_STRING && sToken.type != TK_ID) { code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z); - goto _error; + goto _clean; } str += index; if (sToken.n == 0) { code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z); - goto _error; + goto _clean; } strncpy(pCmd->payload, sToken.z, sToken.n); @@ -1183,7 +1192,7 @@ int tsParseInsertSql(SSqlObj *pSql) { wordexp_t full_path; if (wordexp(pCmd->payload, &full_path, 0) != 0) { code = tscInvalidSQLErrMsg(pCmd->payload, "invalid filename", sToken.z); - goto _error; + goto _clean; } tstrncpy(pCmd->payload, full_path.we_wordv[0], pCmd->allocSize); @@ -1195,7 +1204,7 @@ int tsParseInsertSql(SSqlObj *pSql) { SSchema * pSchema = tscGetTableSchema(pTableMeta); if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { - goto _error; + goto _clean; } SParsedDataColInfo spd = {0}; @@ -1230,7 +1239,7 @@ int tsParseInsertSql(SSqlObj *pSql) { if (spd.hasVal[t] == true) { code = tscInvalidSQLErrMsg(pCmd->payload, "duplicated column name", sToken.z); - goto _error; + goto _clean; } spd.hasVal[t] = true; @@ -1241,13 +1250,13 @@ int tsParseInsertSql(SSqlObj *pSql) { if (!findColumnIndex) { code = tscInvalidSQLErrMsg(pCmd->payload, "invalid column name", sToken.z); - goto _error; + goto _clean; } } if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > tinfo.numOfColumns) { code = tscInvalidSQLErrMsg(pCmd->payload, "column name expected", sToken.z); - goto _error; + goto _clean; } index = 0; @@ -1256,16 +1265,16 @@ int tsParseInsertSql(SSqlObj *pSql) { if (sToken.type != TK_VALUES) { code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES is expected", sToken.z); - goto _error; + goto _clean; } - code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum); + code = doParseInsertStatement(pCmd, &str, &spd, &totalNum); if (code != TSDB_CODE_SUCCESS) { - goto _error; + goto _clean; } } else { code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE are required", sToken.z); - goto _error; + goto _clean; } } @@ -1274,25 +1283,18 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - if (taosArrayGetSize(pCmd->pDataBlocks) > 0) { // merge according to vgId - if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { - goto _error; + if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) { // merge according to vgId + if ((code = tscMergeTableDataBlocks(pSql, true)) != TSDB_CODE_SUCCESS) { + goto _clean; } } code = TSDB_CODE_SUCCESS; goto _clean; -_error: - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - _clean: - taosHashCleanup(pCmd->pTableList); - pCmd->pTableList = NULL; - - pCmd->curSql = NULL; + pCmd->curSql = NULL; pCmd->parseFinished = 1; - return code; } @@ -1309,7 +1311,6 @@ int tsInsertInitialCheck(SSqlObj *pSql) { pCmd->count = 0; pCmd->command = TSDB_SQL_INSERT; - pSql->res.numOfRows = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); @@ -1338,15 +1339,6 @@ int tsParseSql(SSqlObj *pSql, bool initial) { } if (tscIsInsertData(pSql->sqlstr)) { - /* - * Set the fp before parse the sql string, in case of getTableMeta failed, in which - * the error handle callback function can rightfully restore the user-defined callback function (fp). - */ - if (initial && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) { - pSql->fetchFp = pSql->fp; - pSql->fp = (void(*)())tscHandleMultivnodeInsert; - } - if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { return ret; } @@ -1354,10 +1346,11 @@ int tsParseSql(SSqlObj *pSql, bool initial) { // make a backup as tsParseInsertSql may modify the string char* sqlstr = strdup(pSql->sqlstr); ret = tsParseInsertSql(pSql); - if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { + if ((sqlstr == NULL) || (pSql->parseRetry >= 1) || + (ret != TSDB_CODE_TSC_SQL_SYNTAX_ERROR && ret != TSDB_CODE_TSC_INVALID_SQL)) { free(sqlstr); } else { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmd(pCmd, true); free(pSql->sqlstr); pSql->sqlstr = sqlstr; pSql->parseRetry++; @@ -1369,11 +1362,12 @@ int tsParseSql(SSqlObj *pSql, bool initial) { SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmd(pCmd, true); pSql->parseRetry++; ret = tscToSQLCmd(pSql, &SQLInfo); } - SQLInfoDestroy(&SQLInfo); + + SqlInfoDestroy(&SQLInfo); } /* @@ -1399,7 +1393,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL); } - if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = tscMergeTableDataBlocks(pSql, true)) != TSDB_CODE_SUCCESS) { return code; } @@ -1416,27 +1410,38 @@ typedef struct SImportFileSupport { FILE *fp; } SImportFileSupport; -static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { +static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRows) { assert(param != NULL && tres != NULL); + char * tokenBuf = NULL; + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + int32_t count = 0; + int32_t maxRows = 0; + FILE * fp = NULL; + SSqlObj *pSql = tres; SSqlCmd *pCmd = &pSql->cmd; - SImportFileSupport *pSupporter = (SImportFileSupport *) param; + SImportFileSupport *pSupporter = (SImportFileSupport *)param; SSqlObj *pParentSql = pSupporter->pSql; - FILE *fp = pSupporter->fp; + fp = pSupporter->fp; - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // handle error - assert(taos_errno(pSql) == code); + int32_t code = pSql->res.code; - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); - return; + // retry parse data from file and import data from the begining again + if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + assert(pSql->res.numOfRows == 0); + int32_t ret = fseek(fp, 0, SEEK_SET); + if (ret < 0) { + tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); + goto _error; + } + } else if (code != TSDB_CODE_SUCCESS) { + goto _error; } // accumulate the total submit records @@ -1450,25 +1455,32 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns}; tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns); - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; - int32_t count = 0; - int32_t maxRows = 0; + tfree(pCmd->pTableNameList); + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscDestroyBlockArrayList(pSql->cmd.pDataBlocks); - pCmd->pDataBlocks = taosArrayInit(1, POINTER_BYTES); + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (pCmd->pTableBlockHashList == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + } STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock); + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL); if (ret != TSDB_CODE_SUCCESS) { -// return ret; + pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; } - taosArrayPush(pCmd->pDataBlocks, &pTableDataBlock); tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows); - - char *tokenBuf = calloc(1, 4096); + tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); + if (tokenBuf == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { @@ -1496,30 +1508,42 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } tfree(tokenBuf); - free(line); + tfree(line); + + pParentSql->res.code = code; + if (code == TSDB_CODE_SUCCESS) { + if (count > 0) { + code = doPackSendDataBlock(pSql, count, pTableDataBlock); + if (code == TSDB_CODE_SUCCESS) { + return; + } else { + goto _error; + } + } else { + taos_free_result(pSql); + tfree(pSupporter); + fclose(fp); - if (count > 0) { - code = doPackSendDataBlock(pSql, count, pTableDataBlock); - if (code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + pParentSql->fp = pParentSql->fetchFp; + + // all data has been sent to vnode, call user function + int32_t v = (code != TSDB_CODE_SUCCESS) ? code : (int32_t)pParentSql->res.numOfRows; + (*pParentSql->fp)(pParentSql->param, pParentSql, v); return; } + } - } else { - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - pParentSql->fp = pParentSql->fetchFp; +_error: + tfree(tokenBuf); + tfree(line); + taos_free_result(pSql); + tfree(pSupporter); + fclose(fp); - // all data has been sent to vnode, call user function - int32_t v = (pParentSql->res.code != TSDB_CODE_SUCCESS) ? pParentSql->res.code : (int32_t)pParentSql->res.numOfRows; - (*pParentSql->fp)(pParentSql->param, pParentSql, v); - } + tscAsyncResultOnError(pParentSql); } -void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { +void tscImportDataFromFile(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; if (pCmd->command != TSDB_SQL_INSERT) { return; @@ -1529,23 +1553,21 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); - - pNew->cmd.pDataBlocks = taosArrayInit(4, POINTER_BYTES); pCmd->count = 1; - FILE *fp = fopen(pCmd->payload, "r"); + FILE *fp = fopen(pCmd->payload, "rb"); if (fp == NULL) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); - tfree(pSupporter) - tscQueueAsyncRes(pSql); - + tfree(pSupporter); + taos_free_result(pNew); + tscAsyncResultOnError(pSql); return; } pSupporter->pSql = pSql; - pSupporter->fp = fp; + pSupporter->fp = fp; - parseFileSendDataBlock(pSupporter, pNew, 0); + parseFileSendDataBlock(pSupporter, pNew, TSDB_CODE_SUCCESS); } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 30e07294272fa2c8aae823bee428320455d9eb0f..c5f06a52f342b5726321925c1864d58c41afbeb4 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -84,35 +84,35 @@ static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { var->nLen = 0; if (tb->is_null != NULL && *(tb->is_null)) { var->nType = TSDB_DATA_TYPE_NULL; - var->i64Key = 0; + var->i64 = 0; continue; } var->nType = tb->buffer_type; switch (tb->buffer_type) { case TSDB_DATA_TYPE_NULL: - var->i64Key = 0; + var->i64 = 0; break; case TSDB_DATA_TYPE_BOOL: - var->i64Key = (*(int8_t*)tb->buffer) ? 1 : 0; + var->i64 = (*(int8_t*)tb->buffer) ? 1 : 0; break; case TSDB_DATA_TYPE_TINYINT: - var->i64Key = *(int8_t*)tb->buffer; + var->i64 = *(int8_t*)tb->buffer; break; case TSDB_DATA_TYPE_SMALLINT: - var->i64Key = *(int16_t*)tb->buffer; + var->i64 = *(int16_t*)tb->buffer; break; case TSDB_DATA_TYPE_INT: - var->i64Key = *(int32_t*)tb->buffer; + var->i64 = *(int32_t*)tb->buffer; break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - var->i64Key = *(int64_t*)tb->buffer; + var->i64 = *(int64_t*)tb->buffer; break; case TSDB_DATA_TYPE_FLOAT: @@ -219,7 +219,7 @@ static char* normalStmtBuildSql(STscStmt* stmt) { case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: - taosStringBuilderAppendInteger(&sb, var->i64Key); + taosStringBuilderAppendInteger(&sb, var->i64); break; case TSDB_DATA_TYPE_FLOAT: @@ -255,7 +255,6 @@ static char* normalStmtBuildSql(STscStmt* stmt) { //////////////////////////////////////////////////////////////////////////////// // functions for insertion statement preparation - static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { if (bind->is_null != NULL && *(bind->is_null)) { setNull(data + param->offset, param->type, param->bytes); @@ -616,7 +615,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { case TSDB_DATA_TYPE_NCHAR: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_NCHAR: { - size_t output = 0; + int32_t output = 0; if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { return TSDB_CODE_TSC_INVALID_VALUE; } @@ -678,7 +677,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { return TSDB_CODE_SUCCESS; case TSDB_DATA_TYPE_NCHAR: { - size_t output = 0; + int32_t output = 0; if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { return TSDB_CODE_TSC_INVALID_VALUE; } @@ -697,61 +696,44 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; - int32_t alloced = 1, binded = 0; - if (pCmd->batchSize > 0) { - alloced = (pCmd->batchSize + 1) / 2; - binded = pCmd->batchSize / 2; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } - size_t size = taosArrayGetSize(pCmd->pDataBlocks); - for (int32_t i = 0; i < size; ++i) { - STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i); - uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); - uint32_t dataSize = totalDataSize / alloced; - assert(dataSize * alloced == totalDataSize); - - if (alloced == binded) { - totalDataSize += dataSize + sizeof(SSubmitBlk); - if (totalDataSize > pBlock->nAllocSize) { - const double factor = 1.5; - void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - pBlock->pData = (char*)tmp; - pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); - } - } + STableDataBlocks* pBlock = NULL; - char* data = pBlock->pData + sizeof(SSubmitBlk) + dataSize * binded; - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = pBlock->params + j; - int code = doBindParam(data, param, bind + param->idx); - if (code != TSDB_CODE_SUCCESS) { - tscDebug("param %d: type mismatch or invalid", param->idx); - return code; - } - } + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + if (ret != 0) { + // todo handle error } - // actual work of all data blocks is done, update block size and numOfRows. - // note we don't do this block by block during the binding process, because - // we cannot recover if something goes wrong. - pCmd->batchSize = binded * 2 + 1; + uint32_t totalDataSize = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; + if (totalDataSize > pBlock->nAllocSize) { + const double factor = 1.5; - if (binded < alloced) { - return TSDB_CODE_SUCCESS; - } + void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } - size_t total = taosArrayGetSize(pCmd->pDataBlocks); - for (int32_t i = 0; i < total; ++i) { - STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i); + pBlock->pData = (char*)tmp; + pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); + } - uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); - pBlock->size += totalDataSize / alloced; + char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * pCmd->batchSize; + for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { + SParamInfo* param = &pBlock->params[j]; - SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; - pSubmit->numOfRows += pSubmit->numOfRows / alloced; + int code = doBindParam(data, param, &bind[param->idx]); + if (code != TSDB_CODE_SUCCESS) { + tscDebug("param %d: type mismatch or invalid", param->idx); + return code; + } } return TSDB_CODE_SUCCESS; @@ -759,9 +741,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { static int insertStmtAddBatch(STscStmt* stmt) { SSqlCmd* pCmd = &stmt->pSql->cmd; - if ((pCmd->batchSize % 2) == 1) { - ++pCmd->batchSize; - } + ++pCmd->batchSize; return TSDB_CODE_SUCCESS; } @@ -793,50 +773,66 @@ static int insertStmtExecute(STscStmt* stmt) { if (pCmd->batchSize == 0) { return TSDB_CODE_TSC_INVALID_VALUE; } - if ((pCmd->batchSize % 2) == 1) { - ++pCmd->batchSize; - } - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); assert(pCmd->numOfClause == 1); + if (taosHashGetSize(pCmd->pTableBlockHashList) == 0) { + return TSDB_CODE_SUCCESS; + } - if (taosArrayGetSize(pCmd->pDataBlocks) > 0) { - // merge according to vgid - int code = tscMergeTableDataBlocks(stmt->pSql, pCmd->pDataBlocks); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0); - code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); - // set the next sent data vnode index in data block arraylist - pTableMetaInfo->vgroupIndex = 1; - } else { - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } - SSqlObj *pSql = stmt->pSql; - SSqlRes *pRes = &pSql->res; - pRes->numOfRows = 0; - pRes->numOfTotal = 0; - pRes->numOfClauseTotal = 0; + STableDataBlocks* pBlock = NULL; + + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + assert(ret == 0); + pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; + SSubmitBlk* pBlk = (SSubmitBlk*) pBlock->pData; + pBlk->numOfRows = pCmd->batchSize; + pBlk->dataLen = 0; + pBlk->uid = pTableMeta->id.uid; + pBlk->tid = pTableMeta->id.tid; + + int code = tscMergeTableDataBlocks(stmt->pSql, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - pRes->qhandle = 0; + STableDataBlocks* pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0); + code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - pSql->cmd.insertType = 0; - pSql->fetchFp = waitForQueryRsp; - pSql->fp = (void(*)())tscHandleMultivnodeInsert; + SSqlObj* pSql = stmt->pSql; + SSqlRes* pRes = &pSql->res; + pRes->numOfRows = 0; + pRes->numOfTotal = 0; - tscDoQuery(pSql); + tscProcessSql(pSql); // wait for the callback function to post the semaphore tsem_wait(&pSql->rspSem); - return pSql->res.code; + // data block reset + pCmd->batchSize = 0; + for(int32_t i = 0; i < pCmd->numOfTables; ++i) { + if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { + tfree(pCmd->pTableNameList[i]); + } + } + + pCmd->numOfTables = 0; + tfree(pCmd->pTableNameList); + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + + return pSql->res.code; } //////////////////////////////////////////////////////////////////////////////// @@ -867,11 +863,11 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { } tsem_init(&pSql->rspSem, 0, 0); - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->signature = pSql; + pSql->pTscObj = pObj; + pSql->maxRetry = TSDB_MAX_REPLICA; + pStmt->pSql = pSql; - pStmt->pSql = pSql; return pStmt; } @@ -890,7 +886,9 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { SSqlRes *pRes = &pSql->res; pSql->param = (void*) pSql; pSql->fp = waitForQueryRsp; - pSql->cmd.insertType = TSDB_QUERY_TYPE_STMT_INSERT; + pSql->fetchFp = waitForQueryRsp; + + pCmd->insertType = TSDB_QUERY_TYPE_STMT_INSERT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { tscError("%p failed to malloc payload buffer", pSql); @@ -956,8 +954,9 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->isInsert) { return insertStmtBindParam(pStmt, bind); + } else { + return normalStmtBindParam(pStmt, bind); } - return normalStmtBindParam(pStmt, bind); } int taos_stmt_add_batch(TAOS_STMT* stmt) { @@ -981,7 +980,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->isInsert) { ret = insertStmtExecute(pStmt); - } else { + } else { // normal stmt query char* sql = normalStmtBuildSql(pStmt); if (sql == NULL) { ret = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -995,6 +994,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { free(sql); } } + return ret; } diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 18fc79c4748f84d4ec073e4bc678e3a067afb025..9203dcfbbab8d4b512b490bf13ab91fe1b475c22 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -233,6 +233,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { // We extract the lock to tscBuildHeartBeatMsg function. + int64_t now = taosGetTimestampMs(); SSqlObj *pSql = pObj->sqlList; while (pSql) { /* @@ -247,7 +248,8 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql)); pQdesc->stime = htobe64(pSql->stime); pQdesc->queryId = htonl(pSql->queryId); - pQdesc->useconds = htobe64(pSql->res.useconds); + //pQdesc->useconds = htobe64(pSql->res.useconds); + pQdesc->useconds = htobe64(now - pSql->stime); pQdesc->qHandle = htobe64(pSql->res.qhandle); pHeartbeat->numOfQueries++; @@ -262,6 +264,11 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { SSqlStream *pStream = pObj->streamList; while (pStream) { tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql)); + if (pStream->dstTable == NULL) { + pSdesc->dstTable[0] = 0; + } else { + tstrncpy(pSdesc->dstTable, pStream->dstTable, sizeof(pSdesc->dstTable)); + } pSdesc->streamId = htonl(pStream->streamId); pSdesc->num = htobe64(pStream->num); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 91b00e0109b0551b3bde52e326966ff396921019..d5f8f420bf76d9a0b361480fa3255d49ecd9cf52 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -13,13 +13,16 @@ * along with this program. If not, see . */ +#ifndef __APPLE__ #define _BSD_SOURCE #define _XOPEN_SOURCE 500 #define _DEFAULT_SOURCE #define _GNU_SOURCE +#endif // __APPLE__ #include "os.h" -#include "qAst.h" +#include "ttype.h" +#include "texpr.h" #include "taos.h" #include "taosmsg.h" #include "tcompare.h" @@ -40,7 +43,7 @@ #define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } -#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) +#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_BLOCK_DIST_COLUMN_INDEX)) #define TBNAME_LIST_SEP "," typedef struct SColumnList { // todo refactor @@ -59,27 +62,27 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(SArray* pFieldList, int32_t startIdx, const char* name); -static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken); +static char* cloneCurrentDBName(SSqlObj* pSql); static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); -static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); +static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength); -static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult); +static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t convertFunctionId(int32_t optr, int16_t* functionId); static uint8_t convertOptr(SStrToken *pToken); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool intervalQuery); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); @@ -87,7 +90,7 @@ static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuery static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem); +static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem); static int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql); static int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL); @@ -99,8 +102,8 @@ static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryI static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); static int32_t validateEp(char* ep); -static int32_t validateDNodeConfig(tDCLSQL* pOptions); -static int32_t validateLocalConfig(tDCLSQL* pOptions); +static int32_t validateDNodeConfig(SMiscInfo* pOptions); +static int32_t validateLocalConfig(SMiscInfo* pOptions); static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); @@ -109,7 +112,7 @@ static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); -static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); +static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql); static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); @@ -126,6 +129,7 @@ static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid); +static bool validateDebugFlag(int32_t flag); int16_t getNewResColId(SQueryInfo* pQueryInfo) { return pQueryInfo->resColumnId--; @@ -170,6 +174,16 @@ static uint8_t convertOptr(SStrToken *pToken) { } } +static bool validateDebugFlag(int32_t v) { + const static int validFlag[] = {131, 135, 143}; + + for (int i = 0; i < tListLen(validFlag); i++) { + if (v == validFlag[i]) { + return true; + } + } + return false; +} /* * Used during parsing query sql. Since the query sql usually small in length, error position * is not needed in the final error message. @@ -220,7 +234,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (pPwd->n >= TSDB_PASSWORD_LEN) { + if (pPwd->n >= TSDB_KEY_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -228,7 +242,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { } int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { - if (pInfo == NULL || pSql == NULL || pSql->signature != pSql) { + if (pInfo == NULL || pSql == NULL) { return TSDB_CODE_TSC_APP_ERROR; } @@ -238,7 +252,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid || terrno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { terrno = TSDB_CODE_SUCCESS; // clear the error number - return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->pzErrMsg); + return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->msg); } SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); @@ -263,36 +277,48 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_DROP_DB: { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; + const char* msg4 = "table is not super table"; - SStrToken* pzName = &pInfo->pDCLInfo->a[0]; + SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (pInfo->type == TSDB_SQL_DROP_DB) { - assert(pInfo->pDCLInfo->nTokens == 1); - - code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pzName, NULL, NULL); + assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); + code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { - assert(pInfo->pDCLInfo->nTokens == 1); + assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tscSetTableFullName(pTableMetaInfo, pzName, pSql); if(code != TSDB_CODE_SUCCESS) { return code; } + + if (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE) { + code = tscGetTableMeta(pSql, pTableMetaInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + } + } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { pzName->n = strdequote(pzName->z); - strncpy(pTableMetaInfo->name, pzName->z, pzName->n); - } else { // drop user + strncpy(pCmd->payload, pzName->z, pzName->n); + } else { // drop user/account if (pzName->n >= TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - strncpy(pTableMetaInfo->name, pzName->z, pzName->n); + strncpy(pCmd->payload, pzName->z, pzName->n); } break; @@ -300,13 +326,13 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_USE_DB: { const char* msg = "invalid db name"; - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pToken, NULL, NULL); + int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -331,12 +357,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg1 = "invalid db name"; const char* msg2 = "name too long"; - SCreateDBInfo* pCreateDB = &(pInfo->pDCLInfo->dbOpt); + SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); if (tscValidateName(&pCreateDB->dbname) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); + int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname)); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -348,15 +374,15 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { break; } - case TSDB_SQL_CREATE_DNODE: { // todo hostname + case TSDB_SQL_CREATE_DNODE: { const char* msg = "invalid host name (ip address)"; - if (pInfo->pDCLInfo->nTokens > 1) { + if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - SStrToken* pIpAddr = &pInfo->pDCLInfo->a[0]; - pIpAddr->n = strdequote(pIpAddr->z); + SStrToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); + id->n = strdequote(id->z); break; } @@ -366,8 +392,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; - SStrToken* pName = &pInfo->pDCLInfo->user.user; - SStrToken* pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken* pName = &pInfo->pMiscInfo->user.user; + SStrToken* pPwd = &pInfo->pMiscInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -381,7 +407,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SCreateAcctSQL* pAcctOpt = &pInfo->pDCLInfo->acctOpt; + SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; if (pAcctOpt->stat.n > 0) { if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { @@ -396,10 +422,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_DESCRIBE_TABLE: { - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg1 = "invalid table name"; const char* msg2 = "table name too long"; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -417,10 +443,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return tscGetTableMeta(pSql, pTableMetaInfo); } case TSDB_SQL_SHOW_CREATE_TABLE: { - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg1 = "invalid table name"; const char* msg2 = "table name is too long"; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -438,11 +464,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_SHOW_CREATE_DATABASE: { const char* msg1 = "invalid database name"; - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } + if (pToken->n > TSDB_DB_NAME_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -454,29 +481,34 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg3 = "invalid dnode ep"; /* validate the ip address */ - tDCLSQL* pDCL = pInfo->pDCLInfo; + SMiscInfo* pMiscInfo = pInfo->pMiscInfo; /* validate the parameter names and options */ - if (validateDNodeConfig(pDCL) != TSDB_CODE_SUCCESS) { + if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } char* pMsg = pCmd->payload; SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg; - pDCL->a[0].n = strdequote(pDCL->a[0].z); - - strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n); + + SStrToken* t0 = taosArrayGet(pMiscInfo->a, 0); + SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); + + t0->n = strdequote(t0->z); + strncpy(pCfg->ep, t0->z, t0->n); if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n); + strncpy(pCfg->config, t1->z, t1->n); + + if (taosArrayGetSize(pMiscInfo->a) == 3) { + SStrToken* t2 = taosArrayGet(pMiscInfo->a, 2); - if (pDCL->nTokens == 3) { - pCfg->config[pDCL->a[1].n] = ' '; // add sep - strncpy(&pCfg->config[pDCL->a[1].n + 1], pDCL->a[2].z, pDCL->a[2].n); + pCfg->config[t1->n] = ' '; // add sep + strncpy(&pCfg->config[t1->n + 1], t2->z, t2->n); } break; @@ -484,14 +516,14 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_CREATE_USER: case TSDB_SQL_ALTER_USER: { - const char* msg5 = "invalid user rights"; - const char* msg7 = "not support options"; const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; + const char* msg5 = "invalid user rights"; + const char* msg7 = "not support options"; pCmd->command = pInfo->type; - SUserInfo* pUser = &pInfo->pDCLInfo->user; + SUserInfo* pUser = &pInfo->pMiscInfo->user; SStrToken* pName = &pUser->user; SStrToken* pPwd = &pUser->passwd; @@ -535,21 +567,25 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_CFG_LOCAL: { - tDCLSQL* pDCL = pInfo->pDCLInfo; - const char* msg = "invalid configure options or values"; + SMiscInfo *pMiscInfo = pInfo->pMiscInfo; + const char *msg = "invalid configure options or values"; // validate the parameter names and options - if (validateLocalConfig(pDCL) != TSDB_CODE_SUCCESS) { + if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - strncpy(pCmd->payload, pDCL->a[0].z, pDCL->a[0].n); - if (pDCL->nTokens == 2) { - pCmd->payload[pDCL->a[0].n] = ' '; // add sep - strncpy(&pCmd->payload[pDCL->a[0].n + 1], pDCL->a[1].z, pDCL->a[1].n); - } + int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); + assert(numOfToken >= 1 && numOfToken <= 2); - break; + SStrToken* t = taosArrayGet(pMiscInfo->a, 0); + strncpy(pCmd->payload, t->z, t->n); + if (numOfToken == 2) { + SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); + pCmd->payload[t->n] = ' '; // add sep + strncpy(&pCmd->payload[t->n + 1], t1->z, t1->n); + } + return TSDB_CODE_SUCCESS; } case TSDB_SQL_CREATE_TABLE: { @@ -641,7 +677,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } pSql->cmd.parseFinished = 1; - return tscBuildMsg[pCmd->command](pSql, pInfo); + if (tscBuildMsg[pCmd->command] != NULL) { + return tscBuildMsg[pCmd->command](pSql, pInfo); + } else { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); + } } /* @@ -666,6 +706,7 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; const char* msg3 = "sliding cannot be used without interval"; + const char* msg4 = "top/bottom query does not support order by value in interval query"; SSqlCmd* pCmd = &pSql->cmd; @@ -712,6 +753,11 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } + int32_t colId = pQueryInfo->order.orderColId; + if (pQueryInfo->interval.interval > 0 && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + return TSDB_CODE_SUCCESS; } @@ -752,10 +798,10 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ } SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tstrncpy(s.name, aAggs[TSDB_FUNC_TS].aName, sizeof(s.name)); + tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -867,47 +913,54 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableName, SSqlObj* pSql) { const char* msg1 = "name too long"; + const char* msg2 = "acctId too long"; SSqlCmd* pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; - // backup the old name in pTableMetaInfo - char oldName[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(oldName, pTableMetaInfo->name, tListLen(oldName)); + if (hasSpecifyDB(pTableName)) { // db has been specified in sql string so we ignore current db path + code = tNameSetAcctId(&pTableMetaInfo->name, getAccountId(pSql)); + if (code != 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); - if (hasSpecifyDB(pzTableName)) { // db has been specified in sql string so we ignore current db path - code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); + code = tNameFromString(&pTableMetaInfo->name, name, T_NAME_DB|T_NAME_TABLE); if (code != 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } else { // get current DB name first, and then set it into path - SStrToken t = {0}; - getCurrentDBName(pSql, &t); - if (t.n == 0) { // current database not available or not specified - code = TSDB_CODE_TSC_DB_NOT_SELECTED; - } else { - code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); - if (code != 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + char* t = cloneCurrentDBName(pSql); + if (strlen(t) == 0) { + return TSDB_CODE_TSC_DB_NOT_SELECTED; } - } - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tNameFromString(&pTableMetaInfo->name, t, T_NAME_ACCT | T_NAME_DB); + if (code != 0) { + free(t); + return TSDB_CODE_TSC_DB_NOT_SELECTED; + } - /* - * the old name exists and is not equalled to the new name. Release the table meta - * that are corresponding to the old name for the new table name. - */ - if (strlen(oldName) > 0 && strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearTableMetaInfo(pTableMetaInfo, false); + free(t); + + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + + code = tNameFromString(&pTableMetaInfo->name, name, T_NAME_TABLE); + if (code != 0) { + code = invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } - return TSDB_CODE_SUCCESS; + return code; } static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { @@ -944,7 +997,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { return false; } - if (pField->type < TSDB_DATA_TYPE_BOOL || pField->type > TSDB_DATA_TYPE_NCHAR) { + if (!isValidDataType(pField->type)) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } @@ -996,33 +1049,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC return false; } - int32_t nLen = 0; - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - if (p->bytes == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - nLen += p->bytes; - } - - // max tag row length must be less than TSDB_MAX_TAGS_LEN - if (nLen > TSDB_MAX_TAGS_LEN) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - // field name must be unique - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - - if (has(pFieldList, 0, p->name) == true) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - } - /* timestamp in tag is not allowed */ for (int32_t i = 0; i < numOfTags; ++i) { TAOS_FIELD* p = taosArrayGet(pTagsList, i); @@ -1032,7 +1058,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC return false; } - if (p->type < TSDB_DATA_TYPE_BOOL || p->type > TSDB_DATA_TYPE_NCHAR) { + if (!isValidDataType(p->type)) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } @@ -1054,6 +1080,33 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC } } + int32_t nLen = 0; + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + if (p->bytes == 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return false; + } + + nLen += p->bytes; + } + + // max tag row length must be less than TSDB_MAX_TAGS_LEN + if (nLen > TSDB_MAX_TAGS_LEN) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return false; + } + + // field name must be unique + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + + if (has(pFieldList, 0, p->name) == true) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return false; + } + } + return true; } @@ -1091,7 +1144,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { return false; } - if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_NCHAR)) { + if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); return false; } @@ -1155,7 +1208,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { return false; } - if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_NCHAR) { + if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } @@ -1207,9 +1260,12 @@ static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken) { - pDBToken->z = pSql->pTscObj->db; - pDBToken->n = (uint32_t)strlen(pSql->pTscObj->db); +static char* cloneCurrentDBName(SSqlObj* pSql) { + pthread_mutex_lock(&pSql->pTscObj->mutex); + char *p = strdup(pSql->pTscObj->db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); + + return p; } /* length limitation, strstr cannot be applied */ @@ -1235,7 +1291,7 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr /* db name is not specified, the tableName dose not include db name */ if (pDB != NULL) { - if (pDB->n >= TSDB_ACCT_LEN + TSDB_DB_NAME_LEN || pDB->n == 0) { + if (pDB->n >= TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN || pDB->n == 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1274,12 +1330,12 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } -static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; tscColumnListInsert(pQueryInfo->colList, &tsCol); } -static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { +static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSqlExprItem* pItem) { const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; const char* msg2 = "invalid arithmetic expression in select clause"; const char* msg3 = "tag columns can not be used in arithmetic expression"; @@ -1310,7 +1366,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), - -1000, sizeof(double), false); + getNewResColId(pQueryInfo), sizeof(double), false); char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1); @@ -1322,7 +1378,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo, colList, NULL); if (ret != TSDB_CODE_SUCCESS) { taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); + tExprTreeDestroy(pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1331,9 +1387,9 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t for(int32_t k = 0; k < numOfNode; ++k) { SColIndex* pIndex = taosArrayGet(colList, k); if (TSDB_COL_IS_TAG(pIndex->flag)) { - tExprTreeDestroy(&pNode, NULL); + tExprTreeDestroy(pNode, NULL); taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -1356,11 +1412,11 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); // add ts column - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); tbufCloseWriter(&bw); taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); + tExprTreeDestroy(pNode, NULL); } else { columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; @@ -1392,7 +1448,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &pArithExprInfo->uid); if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); + tExprTreeDestroy(pArithExprInfo->pExpr, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } @@ -1420,7 +1476,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t return TSDB_CODE_SUCCESS; } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem) { SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); @@ -1461,7 +1517,7 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { // add the timestamp column into the output columns SColumnIndex index = {0}; // primary timestamp column info int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols); pSupInfo->visible = false; @@ -1469,23 +1525,39 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { +bool isValidDistinctSql(SQueryInfo* pQueryInfo) { + if (pQueryInfo == NULL) { + return false; + } + if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) != TSDB_QUERY_TYPE_STABLE_QUERY) { + return false; + } + if (tscQueryTags(pQueryInfo) && tscSqlExprNumOfExprs(pQueryInfo) == 1){ + return true; + } + return false; +} +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery) { assert(pSelection != NULL && pCmd != NULL); const char* msg2 = "functions can not be mixed up"; const char* msg3 = "not support query expression"; const char* msg5 = "invalid function name"; + const char* msg6 = "only support distinct one tag"; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } - + bool hasDistinct = false; for (int32_t i = 0; i < pSelection->nExpr; ++i) { int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tSQLExprItem* pItem = &pSelection->a[i]; - + tSqlExprItem* pItem = &pSelection->a[i]; + + if (hasDistinct == false) { + hasDistinct = (pItem->distinct == true); + } // project on all fields int32_t optr = pItem->pNode->nSQLOptr; @@ -1519,13 +1591,20 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } + if (hasDistinct == true) { + if (!isValidDistinctSql(pQueryInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + pQueryInfo->distinctTag = true; + } + // there is only one user-defined column in the final result field, add the timestamp column. size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { addPrimaryTsColIntoResult(pQueryInfo); } - if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { + if (!functionCompatibleCheck(pQueryInfo, joinQuery, intervalQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1534,7 +1613,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel * in dealing with super table queries such as: count/first/last */ if (isSTable) { - tscTansformSQLFuncForSTableQuery(pQueryInfo); + tscTansformFuncForSTableQuery(pQueryInfo); if (hasUnsupportFunctionsForSTableQuery(pCmd, pQueryInfo)) { return TSDB_CODE_TSC_INVALID_SQL; @@ -1588,7 +1667,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tabl (functionId == TSDB_FUNC_TAGPRJ)); } -SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { int16_t colId = getNewResColId(pQueryInfo); @@ -1643,7 +1722,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { +int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; @@ -1670,7 +1749,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } // add the primary timestamp column even though it is not required by user - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -1680,7 +1759,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); SSqlExpr* pExpr = - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->numOfParams = 2; @@ -1693,8 +1772,11 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = tGetTableNameColumnSchema(); - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); + SSchema* colSchema = tGetTbnameColumnSchema(); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, colSchema, TSDB_COL_TAG); + } else if (index.columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SSchema colSchema = tGetBlockDistColumnSchema(); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_TAG); } else { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -1708,7 +1790,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } // add the primary timestamp column even though it is not required by user - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1731,7 +1813,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return -1; } else { type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypeDesc[type].nSize; + bytes = tDataTypes[type].bytes; } } else { type = pSchema->type; @@ -1748,7 +1830,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // set reverse order scan data blocks for last query if (functionID == TSDB_FUNC_LAST) { pExpr->numOfParams = 1; - pExpr->param[0].i64Key = TSDB_ORDER_DESC; + pExpr->param[0].i64 = TSDB_ORDER_DESC; pExpr->param[0].nType = TSDB_DATA_TYPE_INT; } @@ -1767,7 +1849,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return TSDB_CODE_SUCCESS; } -void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { +void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { if (pItem->aliasName != NULL) { tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); } else if (multiCols) { @@ -1775,19 +1857,22 @@ void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrT int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); - int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1; - char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0}; - - snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); + if (tsKeepOriginalColumnName) { // keep the original column name + tstrncpy(name, uname, TSDB_COL_NAME_LEN); + } else { + int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1; + char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1] = {0}; + snprintf(tmp, size, "%s(%s)", aAggs[functionId].name, uname); - tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + } } else { // use the user-input result column name int32_t len = MIN(pItem->pNode->token.n + 1, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->pNode->token.z, len); } } -int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { +int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; @@ -1800,6 +1885,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const char* msg7 = "normal table can not apply this function"; const char* msg8 = "multi-columns selection does not support alias column name"; const char* msg9 = "invalid function"; + const char* msg10 = "diff can no be applied to unsigned numeric type"; switch (optr) { case TK_COUNT: { @@ -1817,7 +1903,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - tSQLExprItem* pParamElem = &pItem->pNode->pParam->a[0]; + tSqlExprItem* pParamElem = &pItem->pNode->pParam->a[0]; SStrToken* pToken = &pParamElem->pNode->colInfo; int16_t sqlOptr = pParamElem->pNode->nSQLOptr; if ((pToken->z == NULL || pToken->n == 0) @@ -1834,7 +1920,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } else if (sqlOptr == TK_INTEGER) { // select count(1) from table1 char buf[8] = {0}; @@ -1846,7 +1932,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } if (val == 1) { index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } else { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1866,12 +1952,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col isTag = true; } - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, isTag); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } @@ -1892,7 +1978,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // the time stamp may be always needed if (index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); } return TSDB_CODE_SUCCESS; @@ -1918,7 +2004,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); + tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1927,6 +2013,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if ((getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -1936,8 +2023,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); int16_t colType = pSchema->type; - if (colType <= TSDB_DATA_TYPE_BOOL || colType >= TSDB_DATA_TYPE_BINARY) { + if (!IS_NUMERIC_TYPE(colType)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } else if (IS_UNSIGNED_NUMERIC_TYPE(colType) && optr == TK_DIFF) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); } int16_t resultType = 0; @@ -1962,7 +2051,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col getNewResColId(pQueryInfo), TSDB_KEYSIZE, false); SColumnList ids = getColumnList(1, 0, 0); - insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName, pExpr); + insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); } // functions can not be applied to tags @@ -2005,7 +2094,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } } - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); return TSDB_CODE_SUCCESS; } case TK_FIRST: @@ -2037,7 +2126,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col /* in first/last function, multiple columns can be add to resultset */ for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) { - tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); + tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2096,7 +2185,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, colIndex + i); pExpr->numOfParams = 1; - pExpr->param->i64Key = TSDB_ORDER_ASC; + pExpr->param->i64 = TSDB_ORDER_ASC; break; } @@ -2150,7 +2239,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); + tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); if (pParamElem->pNode->nSQLOptr != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2159,6 +2248,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -2173,7 +2263,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // 2. valid the column type int16_t colType = pSchema[index.columnIndex].type; - if (colType == TSDB_DATA_TYPE_BOOL || colType >= TSDB_DATA_TYPE_BINARY) { + if (!IS_NUMERIC_TYPE(colType)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -2210,7 +2300,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); colIndex += 1; // the first column is ts pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); @@ -2233,12 +2323,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pQueryInfo), TSDB_KEYSIZE, false); - tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].aName, sizeof(pExpr->aliasName)); + tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->aliasName)); const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, - aAggs[TSDB_FUNC_TS].aName, pExpr); + aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts @@ -2309,7 +2399,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SSchema s = {0}; if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - s = tGetTableNameColumnSchema(); + s = *tGetTbnameColumnSchema(); } else { s = pTagSchema[index.columnIndex]; } @@ -2325,7 +2415,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col s.bytes = bytes; TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG); return TSDB_CODE_SUCCESS; } @@ -2350,7 +2440,7 @@ static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t column return columnList; } -void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength) { +void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength) { if (pItem->aliasName != NULL) { strncpy(resultFieldName, pItem->aliasName, nameLength); } else { @@ -2367,6 +2457,14 @@ static bool isTablenameToken(SStrToken* token) { return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } +static bool isTableBlockDistToken(SStrToken* token) { + SStrToken tmpToken = *token; + SStrToken tableToken = {0}; + + extractTableNameFromToken(&tmpToken, &tableToken); + + return (strncasecmp(TSQL_BLOCK_DIST, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_BLOCK_DIST_L)); +} static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; @@ -2396,6 +2494,8 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQu if (isTablenameToken(pToken)) { pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; + } else if (isTableBlockDistToken(pToken)) { + pIndex->columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX; } else if (strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; } else { @@ -2581,10 +2681,10 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg6 = "pattern string is empty"; /* - * database prefix in pInfo->pDCLInfo->a[0] - * wildcard in like clause in pInfo->pDCLInfo->a[1] + * database prefix in pInfo->pMiscInfo->a[0] + * wildcard in like clause in pInfo->pMiscInfo->a[1] */ - SShowInfo* pShowInfo = &pInfo->pDCLInfo->showOpt; + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { // db prefix in tagCond, show table conds in payload @@ -2603,7 +2703,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken, NULL, NULL); + int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -2636,8 +2736,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateIpAddress(pDnodeIp->z, pDnodeIp->n)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - } - + } return TSDB_CODE_SUCCESS; } @@ -2649,7 +2748,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SSqlCmd* pCmd = &pSql->cmd; pCmd->command = pInfo->type; - SStrToken* idStr = &(pInfo->pDCLInfo->ip); + SStrToken* idStr = &(pInfo->pMiscInfo->id); if (idStr->n > TSDB_KILL_MSG_LEN) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2694,7 +2793,7 @@ bool validateIpAddress(const char* ip, size_t size) { return epAddr != INADDR_NONE; } -int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { +int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -2716,9 +2815,9 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex); if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || - (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) || + (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_STDDEV_DST) || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { - if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->param[0].i64Key, &type, &bytes, + if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->param[0].i64, &type, &bytes, &interBytes, 0, true) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2734,7 +2833,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { } /* transfer the field-info back to original input format */ -void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { +void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return; @@ -2758,6 +2857,8 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { functionId = TSDB_FUNC_FIRST; } else if (functionId == TSDB_FUNC_LAST_DST) { functionId = TSDB_FUNC_LAST; + } else if (functionId == TSDB_FUNC_STDDEV_DST) { + functionId = TSDB_FUNC_STDDEV; } getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &pExpr->resType, &pExpr->resBytes, @@ -2774,7 +2875,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_STABLE) == 0) { + if ((aAggs[functionId].status & TSDB_FUNCSTATE_STABLE) == 0) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); return true; } @@ -2801,7 +2902,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool intervalQuery) { int32_t startIdx = 0; size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); @@ -2817,6 +2918,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t factor = functionCompatList[tscSqlExprGet(pQueryInfo, startIdx)->functionId]; + if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { + return false; + } + // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions size_t size = tscSqlExprNumOfExprs(pQueryInfo); @@ -2841,7 +2946,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { } } - if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { + if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { return false; } } @@ -2880,7 +2985,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) STableMeta* pTableMeta = NULL; SSchema* pSchema = NULL; - SSchema s = tscGetTbnameColumnSchema(); int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; @@ -2907,7 +3011,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) int32_t numOfCols = tscGetNumOfColumns(pTableMeta); if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - pSchema = &s; + pSchema = tGetTbnameColumnSchema(); } else { pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); } @@ -3282,7 +3386,12 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); if (pColFilter->filterstr) { - if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { + if (pExpr->nSQLOptr != TK_EQ + && pExpr->nSQLOptr != TK_NE + && pExpr->nSQLOptr != TK_ISNULL + && pExpr->nSQLOptr != TK_NOTNULL + && pExpr->nSQLOptr != TK_LIKE + ) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else { @@ -3347,6 +3456,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) { static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pTableCond, SStringBuilder* sb) { const char* msg0 = "invalid table name list"; + const char* msg1 = "not string following like"; if (pTableCond == NULL) { return TSDB_CODE_SUCCESS; @@ -3364,6 +3474,10 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* if (pTableCond->nSQLOptr == TK_IN) { ret = tablenameListToString(pRight, sb); } else if (pTableCond->nSQLOptr == TK_LIKE) { + if (pRight->nSQLOptr != TK_STRING) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + ret = tablenameCondToString(pRight, sb); } @@ -3398,6 +3512,7 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { const char* msg1 = "invalid join query condition"; + const char* msg2 = "invalid table name in join query"; const char* msg3 = "type of join columns must be identical"; const char* msg4 = "invalid column name in join condition"; @@ -3423,7 +3538,11 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pLeft->uid = pTableMetaInfo->pTableMeta->id.uid; pLeft->tagColId = pTagSchema1->colId; - strcpy(pLeft->tableId, pTableMetaInfo->name); + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pLeft->tableName); + if (code != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -3435,7 +3554,11 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pRight->uid = pTableMetaInfo->pTableMeta->id.uid; pRight->tagColId = pTagSchema2->colId; - strcpy(pRight->tableId, pTableMetaInfo->name); + + code = tNameExtractFullName(&pTableMetaInfo->name, pRight->tableName); + if (code != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } if (pTagSchema1->type != pTagSchema2->type) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -3445,38 +3568,6 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return TSDB_CODE_SUCCESS; } -// todo error handle / such as and /or mixed with +/-/*/ -int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; - - *(*exprString)++ = '('; - - if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { - doArithmeticExprToString(pLeft, exprString); - } else { - int32_t ret = tSQLExprNodeToString(pLeft, exprString); - if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - } - - optrToString(pExpr, exprString); - - if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { - doArithmeticExprToString(pRight, exprString); - } else { - int32_t ret = tSQLExprNodeToString(pRight, exprString); - if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - } - - *(*exprString)++ = ')'; - - return TSDB_CODE_SUCCESS; -} - static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type, uint64_t* uid) { if (pExpr->nSQLOptr == TK_ID) { @@ -3512,7 +3603,7 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL}; + tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL}; // sql function list in selection clause. // Append the sqlExpr into exprList of pQueryInfo structure sequentially @@ -3729,7 +3820,7 @@ static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg return invalidSqlErrMsg(msgBuf, msg); } - *parent = tSQLExprCreate((*parent), pExpr, parentOptr); + *parent = tSqlExprCreate((*parent), pExpr, parentOptr); } else { *parent = pExpr; } @@ -3777,7 +3868,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL * to release expression, e.g., m1.ts = m2.ts, * since this expression is used to set the join query type */ - tSQLExprDestroy(*pExpr); + tSqlExprDestroy(*pExpr); } else { ret = setExprToCond(&pCondExpr->pTimewindow, *pExpr, msg3, parentOptr, pQueryInfo->msg); } @@ -3923,17 +4014,17 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) { if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && ((*pExpr)->nSQLOptr == TK_OR || (*pExpr)->nSQLOptr == TK_AND)) { - tSQLExprNodeDestroy(*pExpr); + tSqlExprNodeDestroy(*pExpr); *pExpr = NULL; } else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) { tSQLExpr* tmpPtr = (*pExpr)->pRight; - tSQLExprNodeDestroy(*pExpr); + tSqlExprNodeDestroy(*pExpr); (*pExpr) = tmpPtr; } else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) { tSQLExpr* tmpPtr = (*pExpr)->pLeft; - tSQLExprNodeDestroy(*pExpr); + tSqlExprNodeDestroy(*pExpr); (*pExpr) = tmpPtr; } @@ -3956,7 +4047,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* (*pExpr) = NULL; } else { - *pOut = tSQLExprCreate(NULL, NULL, (*pExpr)->nSQLOptr); + *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->nSQLOptr); doExtractExprForSTable(pCmd, &(*pExpr)->pLeft, pQueryInfo, &((*pOut)->pLeft), tableIndex); doExtractExprForSTable(pCmd, &(*pExpr)->pRight, pQueryInfo, &((*pOut)->pRight), tableIndex); @@ -4005,14 +4096,13 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pExpr->nSQLOptr == TK_LIKE) { char* str = taosStringBuilderGetResult(sb, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); return TSDB_CODE_SUCCESS; } SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1)); taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - char db[TSDB_TABLE_FNAME_LEN] = {0}; - // remove the duplicated input table names int32_t num = 0; char* tableNameString = taosStringBuilderGetResult(sb, NULL); @@ -4028,7 +4118,8 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, } num = j; - char* name = extractDBName(pTableMetaInfo->name, db); + char name[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(&pTableMetaInfo->name, name); SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; for (int32_t i = 0; i < num; ++i) { @@ -4054,6 +4145,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* str = taosStringBuilderGetResult(&sb1, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); taosStringBuilderDestroy(&sb1); tfree(segments); @@ -4163,23 +4255,23 @@ static int32_t validateJoinExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr static void cleanQueryExpr(SCondExpr* pCondExpr) { if (pCondExpr->pTableCond) { - tSQLExprDestroy(pCondExpr->pTableCond); + tSqlExprDestroy(pCondExpr->pTableCond); } if (pCondExpr->pTagCond) { - tSQLExprDestroy(pCondExpr->pTagCond); + tSqlExprDestroy(pCondExpr->pTagCond); } if (pCondExpr->pColumnCond) { - tSQLExprDestroy(pCondExpr->pColumnCond); + tSqlExprDestroy(pCondExpr->pColumnCond); } if (pCondExpr->pTimewindow) { - tSQLExprDestroy(pCondExpr->pTimewindow); + tSqlExprDestroy(pCondExpr->pTimewindow); } if (pCondExpr->pJoinExpr) { - tSQLExprDestroy(pCondExpr->pJoinExpr); + tSqlExprDestroy(pCondExpr->pJoinExpr); } } @@ -4206,6 +4298,77 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf } } +static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { + const char *msg1 = "invalid tag operator"; + const char* msg2 = "not supported filter condition"; + + do { + if (p->nodeType != TSQL_NODE_EXPR) { + break; + } + + if (!p->_node.pLeft || !p->_node.pRight) { + break; + } + + if (IS_ARITHMETIC_OPTR(p->_node.optr)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + if (!IS_RELATION_OPTR(p->_node.optr)) { + break; + } + + tVariant * vVariant = NULL; + int32_t schemaType = -1; + + if (p->_node.pLeft->nodeType == TSQL_NODE_VALUE && p->_node.pRight->nodeType == TSQL_NODE_COL) { + if (!p->_node.pRight->pSchema) { + break; + } + + vVariant = p->_node.pLeft->pVal; + schemaType = p->_node.pRight->pSchema->type; + } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { + if (!p->_node.pLeft->pSchema) { + break; + } + + vVariant = p->_node.pRight->pVal; + schemaType = p->_node.pLeft->pSchema->type; + } else { + break; + } + + if (schemaType >= TSDB_DATA_TYPE_TINYINT && schemaType <= TSDB_DATA_TYPE_BIGINT) { + schemaType = TSDB_DATA_TYPE_BIGINT; + } else if (schemaType == TSDB_DATA_TYPE_FLOAT || schemaType == TSDB_DATA_TYPE_DOUBLE) { + schemaType = TSDB_DATA_TYPE_DOUBLE; + } + + int32_t retVal = TSDB_CODE_SUCCESS; + if (schemaType == TSDB_DATA_TYPE_BINARY) { + char *tmp = calloc(1, vVariant->nLen + TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { + // pRight->val.nLen + 1 is larger than the actual nchar string length + char *tmp = calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else { + double tmp; + retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); + } + + if (retVal != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + }while (0); + + return TSDB_CODE_SUCCESS; +} + static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; @@ -4247,14 +4410,22 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); doCompactQueryExpr(pExpr); - - tSQLExprDestroy(p1); - tExprTreeDestroy(&p, NULL); + + if (ret == TSDB_CODE_SUCCESS) { + ret = validateTagCondExpr(pCmd, p); + } + + tSqlExprDestroy(p1); + tExprTreeDestroy(p, NULL); taosArrayDestroy(colList); if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); } + + if (ret) { + break; + } } pCondExpr->pTagCond = NULL; @@ -4376,8 +4547,8 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t * * Additional check to avoid data overflow */ - if (pRight->val.i64Key <= INT64_MAX / 1000) { - pRight->val.i64Key *= 1000; + if (pRight->val.i64 <= INT64_MAX / 1000) { + pRight->val.i64 *= 1000; } } else if (pRight->nSQLOptr == TK_FLOAT && timePrecision == TSDB_TIME_PRECISION_MILLI) { pRight->val.dKey *= 1000; @@ -4464,15 +4635,16 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery const char* msg = "illegal value or data overflow"; const char* msg1 = "value is expected"; const char* msg2 = "invalid fill option"; + const char* msg3 = "top/bottom not support fill"; if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - size_t size = tscNumOfFields(pQueryInfo); + size_t numOfFields = tscNumOfFields(pQueryInfo); if (pQueryInfo->fillVal == NULL) { - pQueryInfo->fillVal = calloc(size, sizeof(int64_t)); + pQueryInfo->fillVal = calloc(numOfFields, sizeof(int64_t)); if (pQueryInfo->fillVal == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -4482,19 +4654,21 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery pQueryInfo->fillType = TSDB_FILL_NONE; } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) { pQueryInfo->fillType = TSDB_FILL_NULL; - for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) { + for (int32_t i = START_INTERPO_COL_IDX; i < numOfFields; ++i) { TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); setNull((char*)&pQueryInfo->fillVal[i], pField->type, pField->bytes); } } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) { pQueryInfo->fillType = TSDB_FILL_PREV; + } else if (strncasecmp(pItem->pVar.pz, "next", 4) == 0 && pItem->pVar.nLen == 4) { + pQueryInfo->fillType = TSDB_FILL_NEXT; } else if (strncasecmp(pItem->pVar.pz, "linear", 6) == 0 && pItem->pVar.nLen == 6) { pQueryInfo->fillType = TSDB_FILL_LINEAR; } else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) { pQueryInfo->fillType = TSDB_FILL_SET_VALUE; size_t num = taosArrayGetSize(pFillToken); - if (num == 1) { + if (num == 1) { // no actual value, return with error code return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4505,11 +4679,11 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery if (tscIsPointInterpQuery(pQueryInfo)) { startPos = 0; - if (numOfFillVal > size) { - numOfFillVal = (int32_t)size; + if (numOfFillVal > numOfFields) { + numOfFillVal = (int32_t)numOfFields; } } else { - numOfFillVal = (int16_t)((num > (int32_t)size) ? (int32_t)size : num); + numOfFillVal = (int16_t)((num > (int32_t)numOfFields) ? (int32_t)numOfFields : num); } int32_t j = 1; @@ -4529,10 +4703,10 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery } } - if ((num < size) || ((num - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) { + if ((num < numOfFields) || ((num - 1 < numOfFields) && (tscIsPointInterpQuery(pQueryInfo)))) { tVariantListItem* lastItem = taosArrayGetLast(pFillToken); - for (int32_t i = numOfFillVal; i < size; ++i) { + for (int32_t i = numOfFillVal; i < numOfFields; ++i) { TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) { @@ -4546,6 +4720,14 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + return TSDB_CODE_SUCCESS; } @@ -4569,12 +4751,18 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema) { const char* msg0 = "only support order by primary timestamp"; const char* msg1 = "invalid column name"; - const char* msg2 = "only support order by primary timestamp or queried column"; - const char* msg3 = "only support order by primary timestamp or first tag in groupby clause"; + const char* msg2 = "order by primary timestamp or first tag in groupby clause allowed"; + const char* msg3 = "invalid column in order by clause, only primary timestamp or first tag in groupby clause allowed"; setDefaultOrderInfo(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (pQueryInfo->distinctTag == true) { + pQueryInfo->order.order = TSDB_ORDER_ASC; + pQueryInfo->order.orderColId = 0; + return TSDB_CODE_SUCCESS; + } if (pQuerySql->pSortOrder == NULL) { return TSDB_CODE_SUCCESS; } @@ -4638,7 +4826,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu if (!(orderByTags || orderByTS) && !isTopBottomQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else { + } else { // order by top/bottom result value column is not supported in case of interval query. assert(!(orderByTags && orderByTS)); } @@ -4758,11 +4946,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg17 = "invalid column name"; const char* msg18 = "primary timestamp column cannot be dropped"; const char* msg19 = "invalid new tag name"; + const char* msg20 = "table is not super table"; int32_t code = TSDB_CODE_SUCCESS; SSqlCmd* pCmd = &pSql->cmd; - SAlterTableSQL* pAlterSQL = pInfo->pAlterInfo; + SAlterTableInfo* pAlterSQL = pInfo->pAlterInfo; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, DEFAULT_TABLE_INDEX); @@ -4783,6 +4972,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pAlterSQL->tableType == TSDB_SUPER_TABLE && !(UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg20); + } + if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { @@ -4839,7 +5032,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { SArray* pVarList = pAlterSQL->varList; @@ -4875,14 +5068,14 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name[TSDB_COL_NAME_LEN] = {0}; strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); pItem = taosArrayGet(pVarList, 1); memset(name, 0, tListLen(name)); strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { // Note: update can only be applied to table not super table. @@ -4905,6 +5098,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pItem = taosArrayGet(pVarList, 1); SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); + pAlterSQL->tagData.data = calloc(1, pTagsSchema->bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg13); } @@ -4926,7 +5121,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId); + pUpdateMsg->head.vgId = htonl(pTableMeta->vgId); pUpdateMsg->tid = htonl(pTableMeta->id.tid); pUpdateMsg->uid = htobe64(pTableMeta->id.uid); pUpdateMsg->colId = htons(pTagsSchema->colId); @@ -4955,7 +5150,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t len = 0; if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) { - len = tDataTypeDesc[pTagsSchema->type].nSize; + len = tDataTypes[pTagsSchema->type].bytes; } else { len = varDataTLen(pUpdateMsg->data + schemaLen); } @@ -5002,7 +5197,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[TSDB_COL_NAME_LEN] = {0}; tstrncpy(name1, pItem->pVar.pz, sizeof(name1)); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } @@ -5022,7 +5217,7 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; - if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) { + if (!IS_STREAM_QUERY_VALID(aAggs[functId].status)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -5045,7 +5240,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu bool hasSelectivity = false; for (int32_t j = 0; j < size; ++j) { SSqlExpr* pEx = tscSqlExprGet(pQueryInfo, j); - if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { + if ((aAggs[pEx->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { hasSelectivity = true; break; } @@ -5095,8 +5290,10 @@ int32_t validateEp(char* ep) { return TSDB_CODE_SUCCESS; } -int32_t validateDNodeConfig(tDCLSQL* pOptions) { - if (pOptions->nTokens < 2 || pOptions->nTokens > 3) { +int32_t validateDNodeConfig(SMiscInfo* pOptions) { + int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); + + if (numOfToken < 2 || numOfToken > 3) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -5114,9 +5311,9 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"cqDebugFlag", 11}, }; - SStrToken* pOptionToken = &pOptions->a[1]; + SStrToken* pOptionToken = taosArrayGet(pOptions->a, 1); - if (pOptions->nTokens == 2) { + if (numOfToken == 2) { // reset log and reset query cache does not need value for (int32_t i = 0; i < tokenLogEnd; ++i) { const SDNodeDynConfOption* pOption = &cfgOptions[i]; @@ -5126,7 +5323,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SStrToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 2); int32_t vnodeId = 0; int32_t dnodeId = 0; strdequote(pValToken->z); @@ -5137,14 +5334,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SStrToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 2); int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } return TSDB_CODE_SUCCESS; } else { - SStrToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 2); int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { @@ -5155,8 +5352,8 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { for (int32_t i = tokenDebugFlag; i < tokenDebugFlagEnd; ++i) { const SDNodeDynConfOption* pOption = &cfgOptions[i]; + // options is valid if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - /* options is valid */ return TSDB_CODE_SUCCESS; } } @@ -5165,37 +5362,39 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_TSC_INVALID_SQL; } -int32_t validateLocalConfig(tDCLSQL* pOptions) { - if (pOptions->nTokens < 1 || pOptions->nTokens > 2) { +int32_t validateLocalConfig(SMiscInfo* pOptions) { + int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); + if (numOfToken < 1 || numOfToken > 2) { return TSDB_CODE_TSC_INVALID_SQL; } SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - SStrToken* pOptionToken = &pOptions->a[0]; - if (pOptions->nTokens == 1) { + SStrToken* pOptionToken = taosArrayGet(pOptions->a, 0); + + if (numOfToken == 1) { // reset log does not need value for (int32_t i = 0; i < 1; ++i) { SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { + if ((pOption->len == pOptionToken->n) && + (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { return TSDB_CODE_SUCCESS; } } } else { - SStrToken* pValToken = &pOptions->a[1]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 1); int32_t val = strtol(pValToken->z, NULL, 10); - if (val < 131 || val > 199) { - // options value is out of valid range + if (!validateDebugFlag(val)) { return TSDB_CODE_TSC_INVALID_SQL; } for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) { SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - // options is valid + if ((pOption->len == pOptionToken->n) + && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { return TSDB_CODE_SUCCESS; } } @@ -5218,6 +5417,7 @@ int32_t validateColumnName(char* name) { if (token.type == TK_STRING) { strdequote(token.z); + strntolower(token.z, token.z, token.n); token.n = (uint32_t)strtrim(token.z); int32_t k = tSQLGetToken(token.z, &token.type); @@ -5248,8 +5448,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn const char* msg0 = "soffset/offset can not be less than 0"; const char* msg1 = "slimit/soffset only available for STable query"; - const char* msg2 = "functions mixed up in table query"; - const char* msg3 = "slimit/soffset can not apply to projection query"; + const char* msg2 = "slimit/soffset can not apply to projection query"; // handle the limit offset value, validate the limit pQueryInfo->limit = pQuerySql->limit; @@ -5274,7 +5473,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } // for projection query on super table, all queries are subqueries @@ -5312,7 +5511,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn // keep original limitation value in globalLimit pQueryInfo->clauseLimit = pQueryInfo->limit.limit; pQueryInfo->prjOffset = pQueryInfo->limit.offset; - pQueryInfo->tableLimit = -1; + pQueryInfo->vgroupLimit = -1; if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { /* @@ -5322,7 +5521,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn * than or equal to the value of limit. */ if (pQueryInfo->limit.limit > 0) { - pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset; + pQueryInfo->vgroupLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset; pQueryInfo->limit.limit = -1; } @@ -5332,30 +5531,12 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn if (pQueryInfo->slimit.limit != -1 || pQueryInfo->slimit.offset != 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - bool hasTags = false; - bool hasOtherFunc = false; - // filter the query functions operating on "tbname" column that are not supported by normal columns. - for (int32_t i = 0; i < size; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { - hasTags = true; - } else { - hasOtherFunc = true; - } - } - - if (hasTags && hasOtherFunc) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } } return TSDB_CODE_SUCCESS; } -static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { const char* msg = "invalid number of options"; pMsg->daysToKeep = htonl(-1); @@ -5368,22 +5549,22 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* p tVariantListItem* p0 = taosArrayGet(pKeep, 0); switch (s) { case 1: { - pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key); + pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64); } break; case 2: { tVariantListItem* p1 = taosArrayGet(pKeep, 1); - pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key); - pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key); + pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64); + pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); break; } case 3: { tVariantListItem* p1 = taosArrayGet(pKeep, 1); tVariantListItem* p2 = taosArrayGet(pKeep, 2); - pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key); - pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key); - pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64Key); + pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64); + pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); + pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); break; } default: { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5393,7 +5574,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* p return TSDB_CODE_SUCCESS; } -static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { +static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDbInfo) { const char* msg = "invalid time precision"; pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default @@ -5417,7 +5598,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo return TSDB_CODE_SUCCESS; } -static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { pMsg->maxTables = htonl(-1); // max tables can not be set anymore pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); @@ -5432,9 +5613,10 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->quorum = pCreateDb->quorum; pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->update = pCreateDb->update; + pMsg->cacheLastRow = pCreateDb->cachelast; } -int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { +int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql) { SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload); setCreateDBOption(pMsg, pCreateDbSql); @@ -5513,7 +5695,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex}; - tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); int32_t numOfFields = tscNumOfFields(pQueryInfo); SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1); @@ -5548,7 +5730,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { if ((pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) && !(pExpr->functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) { SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; - getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, (int32_t)pExpr->param[0].i64Key, &pExpr->resType, + getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, (int32_t)pExpr->param[0].i64, &pExpr->resType, &pExpr->resBytes, &pExpr->interBytes, tagLength, isSTable); } } @@ -5677,7 +5859,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) continue; } - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { numOfSelectivity++; } else { numOfAggregation++; @@ -5709,7 +5891,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int16_t functionId = pExpr->functionId; - if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) { + if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == 0) { continue; } @@ -5752,7 +5934,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema s = tGetTableNameColumnSchema(); + SSchema s = *tGetTbnameColumnSchema(); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); int16_t bytes = 0; int16_t type = 0; @@ -5820,14 +6002,43 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo return TSDB_CODE_SUCCESS; } +static int32_t doTagFunctionCheck(SQueryInfo* pQueryInfo) { + bool tagProjection = false; + bool tableCounting = false; + + int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < numOfCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + int32_t functionId = pExpr->functionId; + + if (functionId == TSDB_FUNC_TAGPRJ) { + tagProjection = true; + continue; + } + + if (functionId == TSDB_FUNC_COUNT) { + assert(pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX); + tableCounting = true; + } + } + + return (tableCounting && tagProjection)? -1:0; +} + int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { const char* msg1 = "functions/columns not allowed in group by query"; const char* msg2 = "projection query on columns not allowed"; const char* msg3 = "group by not allowed on projection query"; const char* msg4 = "retrieve tags not compatible with group by or interval query"; + const char* msg5 = "functions can not be mixed up"; // only retrieve tags, group by is not supportted if (tscQueryTags(pQueryInfo)) { + if (doTagFunctionCheck(pQueryInfo) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } + if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->interval.interval > 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } else { @@ -5867,7 +6078,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } } - if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && + if (IS_MULTIOUTPUT(aAggs[functId].status) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_PRJ) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5908,25 +6119,33 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ if (pExprList->nExpr != 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - + bool server_status = false; tSQLExpr* pExpr = pExprList->a[0].pNode; if (pExpr->operand.z == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - + //handle 'select 1' + if (pExpr->token.n == 1 && 0 == strncasecmp(pExpr->token.z, "1", 1)) { + server_status = true; + } else { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + } // TODO redefine the function - SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, - {"server_version()", 16}, - {"server_status()", 15}, - {"client_version()", 16}, - {"current_user()", 14}}; + SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, + {"server_version()", 16}, + {"server_status()", 15}, + {"client_version()", 16}, + {"current_user()", 14}}; int32_t index = -1; - for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { - if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && - functionsInfo[i].len == pExpr->operand.n) { - index = i; - break; + if (server_status == true) { + index = 2; + } else { + for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { + if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && + functionsInfo[i].len == pExpr->operand.n) { + index = i; + break; + } } } @@ -5946,7 +6165,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ SColumnIndex ind = {0}; SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, getNewResColId(pQueryInfo), tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false); + tDataTypes[TSDB_DATA_TYPE_INT].bytes, getNewResColId(pQueryInfo), tDataTypes[TSDB_DATA_TYPE_INT].bytes, false); const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; tstrncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); @@ -5971,9 +6190,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { } if (pCreate->quorum != -1 && - (pCreate->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCreate->quorum > TSDB_MAX_DB_REPLICA_OPTION)) { + (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, - TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); + TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -6049,8 +6268,10 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { char tmpBuf[1024] = {0}; int32_t tmpLen = 0; tmpLen = - sprintf(tmpBuf, "%s(uid:%" PRId64 ", %d)", aAggs[pExpr->functionId].aName, pExpr->uid, pExpr->colInfo.colId); - if (tmpLen + offset > totalBufSize) break; + sprintf(tmpBuf, "%s(uid:%" PRId64 ", %d)", aAggs[pExpr->functionId].name, pExpr->uid, pExpr->colInfo.colId); + + if (tmpLen + offset >= totalBufSize - 1) break; + offset += sprintf(str + offset, "%s", tmpBuf); @@ -6059,7 +6280,9 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { } } + assert(offset < totalBufSize); str[offset] = ']'; + assert(offset < totalBufSize); tscDebug("%p select clause:%s", pSql, str); } @@ -6136,96 +6359,186 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { const int32_t TABLE_INDEX = 0; const int32_t STABLE_INDEX = 1; - STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); + STableMetaInfo* pStableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst - SStrToken* pToken = &(pCreateTable->usingInfo.stableName); + int32_t numOfTables = (int32_t) taosArrayGetSize(pCreateTable->childTableInfo); + for(int32_t j = 0; j < numOfTables; ++j) { + SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + SStrToken* pToken = &pCreateTableInfo->stableName; + if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } - int32_t code = tscSetTableFullName(pStableMeterMetaInfo, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = tscSetTableFullName(pStableMetaInfo, pToken, pSql); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - // get meter meta from mnode - tstrncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, sizeof(pCreateTable->usingInfo.tagdata.name)); - SArray* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals; + // get table meta from mnode + code = tNameExtractFullName(&pStableMetaInfo->name, pCreateTableInfo->tagdata.name); - code = tscGetTableMeta(pSql, pStableMeterMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + SArray* pValList = pCreateTableInfo->pTagVals; + code = tscGetTableMeta(pSql, pStableMetaInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - size_t size = taosArrayGetSize(pList); - if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != size) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); - } + size_t valSize = taosArrayGetSize(pValList); - // too long tag values will return invalid sql, not be truncated automatically - SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta); - STagData* pTag = &pCreateTable->usingInfo.tagdata; - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } + // too long tag values will return invalid sql, not be truncated automatically + SSchema *pTagSchema = tscGetTableTagSchema(pStableMetaInfo->pTableMeta); + STagData *pTag = &pCreateTableInfo->tagdata; + + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } - int32_t ret = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < size; ++i) { - SSchema* pSchema = &pTagSchema[i]; - tVariantListItem* pItem = taosArrayGet(pList, i); - char tagVal[TSDB_MAX_TAGS_LEN]; - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->pVar.nLen > pSchema->bytes) { + SArray* pNameList = NULL; + size_t nameSize = 0; + int32_t schemaSize = tscGetNumOfTags(pStableMetaInfo->pTableMeta); + int32_t ret = TSDB_CODE_SUCCESS; + + if (pCreateTableInfo->pTagNames) { + pNameList = pCreateTableInfo->pTagNames; + nameSize = taosArrayGetSize(pNameList); + + if (valSize != nameSize) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } + + if (schemaSize < valSize) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - } - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + bool findColumnIndex = false; - // check again after the convert since it may be converted from binary to nchar. - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - int16_t len = varDataTLen(tagVal); - if (len > pSchema->bytes) { + for (int32_t i = 0; i < nameSize; ++i) { + SStrToken* sToken = taosArrayGet(pNameList, i); + if (TK_STRING == sToken->type) { + tscDequoteAndTrimToken(sToken); + } + + tVariantListItem* pItem = taosArrayGet(pValList, i); + + findColumnIndex = false; + + // todo speedup by using hash list + for (int32_t t = 0; t < schemaSize; ++t) { + if (strncmp(sToken->z, pTagSchema[t].name, sToken->n) == 0 && strlen(pTagSchema[t].name) == sToken->n) { + SSchema* pSchema = &pTagSchema[t]; + + char tagVal[TSDB_MAX_TAGS_LEN]; + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + if (pItem->pVar.nLen > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + + // check again after the convert since it may be converted from binary to nchar. + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + int16_t len = varDataTLen(tagVal); + if (len > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + if (ret != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); + + findColumnIndex = true; + break; + } + } + + if (!findColumnIndex) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return tscInvalidSQLErrMsg(pCmd->payload, "invalid tag name", sToken->z); + } + } + } else { + if (schemaSize != valSize) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - } - if (ret != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + for (int32_t i = 0; i < valSize; ++i) { + SSchema* pSchema = &pTagSchema[i]; + tVariantListItem* pItem = taosArrayGet(pValList, i); + + char tagVal[TSDB_MAX_TAGS_LEN]; + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + if (pItem->pVar.nLen > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + + // check again after the convert since it may be converted from binary to nchar. + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + int16_t len = varDataTLen(tagVal); + if (len > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + if (ret != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); + } } + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + tdDestroyKVRowBuilder(&kvRowBuilder); + if (row == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tdSortKVRowByColIdx(row); + pTag->dataLen = kvRowLen(row); + if (pTag->data == NULL) { + pTag->data = malloc(pTag->dataLen); + } - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - } + kvRowCpy(pTag->data, row); + free(row); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); - if (row == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - tdSortKVRowByColIdx(row); - pTag->dataLen = kvRowLen(row); - kvRowCpy(pTag->data, row); - free(row); + // table name + if (tscValidateName(&(pCreateTableInfo->name)) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } - // table name - if (tscValidateName(&pInfo->pCreateTableInfo->name) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); + ret = tscSetTableFullName(pTableMetaInfo, &pCreateTableInfo->name, pSql); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } - STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - ret = tscSetTableFullName(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + pCreateTableInfo->fullname = calloc(1, tNameLen(&pTableMetaInfo->name) + 1); + ret = tNameExtractFullName(&pTableMetaInfo->name, pCreateTableInfo->fullname); + if (ret != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } return TSDB_CODE_SUCCESS; @@ -6233,10 +6546,12 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { const char* msg1 = "invalid table name"; + const char* msg2 = "functions not allowed in CQ"; const char* msg3 = "fill only available for interval query"; const char* msg4 = "fill option not supported in stream computing"; const char* msg5 = "sql too long"; // todo ADD support const char* msg6 = "from missing in subclause"; + const char* msg7 = "time interval is required"; SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); @@ -6246,10 +6561,10 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db - SStrToken* pzTableName = &(pCreateTable->name); + SStrToken* pName = &(pCreateTable->name); SQuerySQL* pQuerySql = pCreateTable->pSelect; - if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { + if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -6275,7 +6590,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6288,15 +6603,19 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { // set interval value if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; - } else { - if ((pQueryInfo->interval.interval > 0) && - (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { - return TSDB_CODE_TSC_INVALID_SQL; - } + } + + if ((pQueryInfo->interval.interval > 0) && + (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + if (!tscIsProjectionQuery(pQueryInfo) && pQueryInfo->interval.interval == 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } // set the created table[stream] name - code = tscSetTableFullName(pTableMetaInfo, pzTableName, pSql); + code = tscSetTableFullName(pTableMetaInfo, pName, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -6338,18 +6657,52 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_SUCCESS; } +static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { + const char* msg3 = "start(end) time of query range required or time range too large"; + + if (pQueryInfo->interval.interval == 0) { + return TSDB_CODE_SUCCESS; + } + + bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); + if (initialWindows) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + + int64_t intervalRange = 0; + if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { + int64_t f = 1; + if (pQueryInfo->interval.intervalUnit == 'n') { + f = 30L * MILLISECOND_PER_DAY; + } else if (pQueryInfo->interval.intervalUnit == 'y') { + f = 365L * MILLISECOND_PER_DAY; + } + + intervalRange = pQueryInfo->interval.interval * f; + } else { + intervalRange = pQueryInfo->interval.interval; + } + // number of result is not greater than 10,000,000 + if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + return TSDB_CODE_SUCCESS; +} + int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0)); const char* msg0 = "invalid table name"; - const char* msg2 = "point interpolation query needs timestamp"; - const char* msg5 = "fill only available for interval query"; - const char* msg6 = "start(end) time of query range required or time range too large"; - const char* msg7 = "illegal number of tables in from clause"; - const char* msg8 = "too many columns in selection clause"; - const char* msg9 = "TWA query requires both the start and end time"; - const char* msg10 = "too many tables in from clause"; - const char* msg11 = "invalid table alias name"; + const char* msg1 = "point interpolation query needs timestamp"; + const char* msg2 = "fill only available for interval query"; + const char* msg3 = "start(end) time of query range required or time range too large"; + const char* msg4 = "illegal number of tables in from clause"; + const char* msg5 = "too many columns in selection clause"; + const char* msg6 = "too many tables in from clause"; + const char* msg7 = "invalid table alias name"; int32_t code = TSDB_CODE_SUCCESS; @@ -6365,7 +6718,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // too many result columns not support order by in query if (pQuerySql->pSelection->nExpr > TSDB_MAX_COLUMNS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } /* @@ -6383,13 +6736,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { size_t fromSize = taosArrayGetSize(pQuerySql->from); if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } pQueryInfo->command = TSDB_SQL_SELECT; if (fromSize > 4) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } // set all query tables, which are maybe more than one. @@ -6422,17 +6775,17 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { tVariantListItem* p1 = taosArrayGet(pQuerySql->from, i + 1); if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING}; if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } // has no table alias name if (memcmp(pTableItem->pz, p1->pVar.pz, p1->pVar.nLen) == 0) { - extractTableName(pTableMetaInfo1->name, pTableMetaInfo1->aliasName); + strncpy(pTableMetaInfo1->aliasName, tNameGetTableName(&pTableMetaInfo1->name), tListLen(pTableMetaInfo->aliasName)); } else { tstrncpy(pTableMetaInfo1->aliasName, p1->pVar.pz, sizeof(pTableMetaInfo1->aliasName)); } @@ -6486,7 +6839,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { int32_t joinQuery = (pQuerySql->from != NULL && taosArrayGetSize(pQuerySql->from) > 2); - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { + int32_t intervalQuery = !(pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0); + + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery, intervalQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6505,12 +6860,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } } - // user does not specified the query time window, twa is not allowed in such case. - if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX || - (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - // no result due to invalid query time range if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; @@ -6518,7 +6867,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } if (!hasTimestampForPointInterpQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } // in case of join query, time range is required. @@ -6526,7 +6875,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); if (timeRange == 0 && pQueryInfo->window.skey == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -6540,31 +6889,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { tscFieldInfoUpdateOffset(pQueryInfo); - /* - * fill options are set at the end position, when all columns are set properly - * the columns may be increased due to group by operation - */ if (pQuerySql->fillType != NULL) { if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (pQueryInfo->interval.interval > 0 && pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { - bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); - if (initialWindows) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->interval.interval) > MAX_INTERVAL_TIME_WINDOW) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); - } + /* + * fill options are set at the end position, when all columns are set properly + * the columns may be increased due to group by operation + */ + if ((code = checkQueryRangeForFill(pCmd, pQueryInfo)) != TSDB_CODE_SUCCESS) { + return code; } - int32_t ret = parseFillClause(pCmd, pQueryInfo, pQuerySql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if ((code = parseFillClause(pCmd, pQueryInfo, pQuerySql)) != TSDB_CODE_SUCCESS) { + return code; } } @@ -6675,7 +7014,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS // check for dividing by 0 if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { if (pRight->nodeType == TSQL_NODE_VALUE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64Key == 0) { + if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64 == 0) { return TSDB_CODE_TSC_INVALID_SQL; } else if (pRight->pVal->nType == TSDB_DATA_TYPE_FLOAT && pRight->pVal->dKey == 0) { return TSDB_CODE_TSC_INVALID_SQL; @@ -6707,3 +7046,4 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { return false; } + diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index fcc93ffadc4f7dc62b79fffe245947799a770b40..67352ca71cab03bb9cd2f6b920b97abebbe1420a 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -66,50 +66,6 @@ STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { return pTableMeta->tableInfo; } -bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) { - if (!VALIDNUMOFCOLS(numOfCols)) { - return false; - } - - /* first column must be the timestamp, which is a primary key */ - if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { - return false; - } - - /* type is valid, length is valid */ - int32_t rowLen = 0; - - for (int32_t i = 0; i < numOfCols; ++i) { - // 1. valid types - if (pSchema[i].type > TSDB_DATA_TYPE_TIMESTAMP || pSchema[i].type < TSDB_DATA_TYPE_BOOL) { - return false; - } - - // 2. valid length for each type - if (pSchema[i].type == TSDB_DATA_TYPE_TIMESTAMP) { - if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { - return false; - } - } else { - if (pSchema[i].bytes != tDataTypeDesc[pSchema[i].type].nSize) { - return false; - } - } - - // 3. valid column names - for (int32_t j = i + 1; j < numOfCols; ++j) { - if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { - return false; - } - } - - rowLen += pSchema[i].bytes; - } - - // valid total length - return (rowLen <= TSDB_MAX_BYTES_PER_ROW); -} - SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { assert(pTableMeta != NULL); @@ -130,23 +86,15 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { return NULL; } -static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) { - corVgroupInfo->version = 0; - corVgroupInfo->inUse = 0; - corVgroupInfo->numOfEps = vgroupInfo->numOfEps; - for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) { - corVgroupInfo->epAddr[i].fqdn = strdup(vgroupInfo->epAddr[i].fqdn); - corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port; - } -} - -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { - assert(pTableMetaMsg != NULL); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2 && pTableMetaMsg->numOfTags >= 0); int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); + pTableMeta->tableType = pTableMetaMsg->tableType; - + pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, .precision = pTableMetaMsg->precision, @@ -156,22 +104,10 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->id.tid = pTableMetaMsg->tid; pTableMeta->id.uid = pTableMetaMsg->uid; - SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; - pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps; - pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId; - - for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - SEpAddrMsg* pEpMsg = &pTableMetaMsg->vgroup.epAddr[i]; - - pVgroupInfo->epAddr[i].fqdn = strndup(pEpMsg->fqdn, tListLen(pEpMsg->fqdn)); - pVgroupInfo->epAddr[i].port = pEpMsg->port; - } - - tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo); - pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; - tstrncpy(pTableMeta->sTableId, pTableMetaMsg->sTableId, TSDB_TABLE_FNAME_LEN); + + tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN); memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); @@ -180,13 +116,44 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - if (size != NULL) { - *size = sizeof(STableMeta) + schemaSize; - } - return pTableMeta; } +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) { + assert(pExisted != NULL && src != NULL); + if (pExisted->numOfEps != src->numOfEps) { + return false; + } + + for(int32_t i = 0; i < pExisted->numOfEps; ++i) { + if (pExisted->ep[i].port != src->epAddr[i].port) { + return false; + } + + if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) { + return false; + } + } + + return true; +} + +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { + assert(pVgroupMsg != NULL); + + SNewVgroupInfo info = {0}; + info.numOfEps = pVgroupMsg->numOfEps; + info.vgId = pVgroupMsg->vgId; + info.inUse = 0; + + for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) { + tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN); + info.ep[i].port = pVgroupMsg->epAddr[i].port; + } + + return info; +} + // todo refactor UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 59bcdd691d6a8cfe2f2e65da0c87da9c232fee57..d005eaf75cb7544d9ab51cd313a542cf04da53e3 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -14,7 +14,6 @@ */ #include "os.h" -#include "tcache.h" #include "tcmdtype.h" #include "trpc.h" #include "tscLocalMerge.h" @@ -26,8 +25,6 @@ #include "ttimer.h" #include "tlockfree.h" -///SRpcCorEpSet tscMgmtEpSet; - int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); @@ -45,32 +42,30 @@ static int32_t getWaitingTimeInterval(int32_t count) { return 0; } - return initial * (2<<(count - 2)); + return initial * ((2u)<<(count - 2)); } -static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { - assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); - - SRpcEpSet* pEpSet = &pSql->epSet; +static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupInfo* pVgroupInfo) { + assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); // Issue the query to one of the vnode among a vgroup randomly. // change the inUse property would not affect the isUse attribute of STableMeta pEpSet->inUse = rand() % pVgroupInfo->numOfEps; // apply the FQDN string length check here - bool hasFqdn = false; + bool existed = false; pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; - if (!hasFqdn) { - hasFqdn = (strlen(pEpSet->fqdn[i]) > 0); + int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + if (len > 0) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + existed = true; } } - - assert(hasFqdn); + assert(existed); } static void tscDumpMgmtEpSet(SSqlObj *pSql) { @@ -87,7 +82,8 @@ static void tscEpSetHtons(SRpcEpSet *s) { bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; - } + } + for (int32_t i = 0; i < s1->numOfEps; i++) { if (s1->port[i] != s2->port[i] || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) @@ -95,6 +91,7 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { } return true; } + void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { // no need to update if equal SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; @@ -102,37 +99,45 @@ void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { pCorEpSet->epSet = *pEpSet; taosCorEndWrite(&pCorEpSet->version); } -static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { + +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { if (pVgroupInfo == NULL) { return;} - taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pEpSet->numOfEps = pVgroupInfo->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; } - taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } - taosCorBeginWrite(&pVgroupInfo->version); - tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); - pVgroupInfo->inUse = pEpSet->inUse; - pVgroupInfo->numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - tfree(pVgroupInfo->epAddr[i].fqdn); - pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i])); - pVgroupInfo->epAddr[i].port = pEpSet->port[i]; + int32_t vgId = pTableMetaInfo->pTableMeta->vgId; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + assert(vgId == 0); + return; } - tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); - taosCorEndWrite(&pVgroupInfo->version); + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); + + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; + } + + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -147,11 +152,20 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SSqlObj *pSql = tres; SSqlRes *pRes = &pSql->res; - if (code == 0) { + if (code == TSDB_CODE_SUCCESS) { SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; - SRpcEpSet * epSet = &pRsp->epSet; + SRpcEpSet *epSet = &pRsp->epSet; if (epSet->numOfEps > 0) { tscEpSetHtons(epSet); + + //SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { + // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); + // for (int8_t i = 0; i < epSet->numOfEps; i++) { + // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); + // } + //} + //concurrency problem, update mgmt epset anyway tscUpdateMgmtEpSet(pSql, epSet); } @@ -161,11 +175,40 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { tscKillConnection(pObj); return; } else { - if (pRsp->queryId) tscKillQuery(pObj, htonl(pRsp->queryId)); - if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId)); + if (pRsp->queryId) { + tscKillQuery(pObj, htonl(pRsp->queryId)); + } + + if (pRsp->streamId) { + tscKillStream(pObj, htonl(pRsp->streamId)); + } } + + int32_t total = htonl(pRsp->totalDnodes); + int32_t online = htonl(pRsp->onlineDnodes); + assert(online <= total); + + if (online < total) { + tscError("HB:%p, total dnode:%d, online dnode:%d", pSql, total, online); + pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[0] = total; + pRes->length[1] = online; } else { tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[1] = 0; + if (pRes->length[0] == 0) { + pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node + } } if (pObj->hbrid != 0) { @@ -230,7 +273,8 @@ int tscSendMsgToServer(SSqlObj *pSql) { .code = 0 }; - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); + + rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); return TSDB_CODE_SUCCESS; } @@ -252,8 +296,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); - taosRemoveRef(tscObjRef, pSql->self); - taosReleaseRef(tscObjRef, pSql->self); + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); return; } @@ -263,8 +307,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - taosRemoveRef(tscObjRef, pSql->self); - taosReleaseRef(tscObjRef, pSql->self); + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); return; } @@ -280,24 +324,27 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } int32_t cmd = pCmd->command; - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_INSERT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.submitSchema = 1; + } + + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || - rpcMsg->code == TSDB_CODE_APP_NOT_READY || - rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { - tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.submitSchema = 1; - } + rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { + + pSql->retry++; + tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), pSql->retry); pSql->res.code = rpcMsg->code; // keep the previous error code if (pSql->retry > pSql->maxRetry) { tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); } else { - // wait for a little bit moment and then retry, todo do not sleep in rpc callback thread + // wait for a little bit moment and then retry + // todo do not sleep in rpc callback thread, add this process into queueu to process if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { int32_t duration = getWaitingTimeInterval(pSql->retry); taosMsleep(duration); @@ -307,7 +354,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { // if there is an error occurring, proceed to the following error handling procedure. if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); + taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); return; } @@ -371,17 +418,22 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { bool shouldFree = tscShouldBeFreed(pSql); if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + pRes->code = rpcMsg->code; + } rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; (*pSql->fp)(pSql->param, pSql, rpcMsg->code); } - taosReleaseRef(tscObjRef, pSql->self); + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - taosRemoveRef(tscObjRef, pSql->self); - tscDebug("%p sqlObj is automatically freed", pSql); + taosRemoveRef(tscObjRef, handle); + tscDebug("%p sqlObj is automatically freed", pSql); } + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); } @@ -401,8 +453,8 @@ int doProcessSql(SSqlObj *pSql) { } if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); - return pRes->code; + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; } int32_t code = tscSendMsgToServer(pSql); @@ -410,24 +462,28 @@ int doProcessSql(SSqlObj *pSql) { // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. if (code != TSDB_CODE_SUCCESS) { pRes->code = code; - tscQueueAsyncRes(pSql); - return code; + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; } return TSDB_CODE_SUCCESS; } int tscProcessSql(SSqlObj *pSql) { - char *name = NULL; + char name[TSDB_TABLE_FNAME_LEN] = {0}; + SSqlCmd *pCmd = &pSql->cmd; - + uint32_t type = 0; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = NULL; - uint32_t type = 0; if (pQueryInfo != NULL) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - name = (pTableMetaInfo != NULL)? pTableMetaInfo->name:NULL; + if (pTableMetaInfo != NULL) { + tNameExtractFullName(&pTableMetaInfo->name, name); + } + type = pQueryInfo->type; // while numOfTables equals to 0, it must be Heartbeat @@ -440,9 +496,9 @@ int tscProcessSql(SSqlObj *pSql) { pSql->res.code = TSDB_CODE_TSC_APP_ERROR; return pSql->res.code; } - } else if (pCmd->command < TSDB_SQL_LOCAL) { + } else if (pCmd->command >= TSDB_SQL_LOCAL) { //pSql->epSet = tscMgmtEpSet; - } else { // local handler +// } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -451,10 +507,10 @@ int tscProcessSql(SSqlObj *pSql) { int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; - pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); - pRetrieveMsg->free = htons(pQueryInfo->type); + pRetrieveMsg->free = htons(pQueryInfo->type); + pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle); // todo valid the vgroupId at the client side STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -478,8 +534,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); - tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId); + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); + tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgId); } pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); @@ -498,7 +554,6 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // NOTE: shell message size should not include SMsgDesc int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); - int32_t vgId = pTableMeta->vgroupInfo.vgId; SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; pMsgDesc->numOfVnodes = htonl(1); // always one vnode @@ -506,7 +561,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(SMsgDesc); SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; - pShellMsg->header.vgId = htonl(vgId); + pShellMsg->header.vgId = htonl(pTableMeta->vgId); pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc pShellMsg->length = pShellMsg->header.contLen; @@ -514,9 +569,12 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epSet); - tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, pTableMeta->vgId, pSql->cmd.numOfTablesInSubmit, pSql->epSet.numOfEps); return TSDB_CODE_SUCCESS; } @@ -524,9 +582,11 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { /* * for table query, simply return the size <= 1k */ -static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); @@ -534,6 +594,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs * 2); int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + int32_t tableSerialize = 0; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -550,7 +612,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + - tableSerialize + 4096; + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { @@ -560,24 +622,31 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - SVgroupInfo* pVgroupInfo = NULL; + int32_t vgId = -1; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t index = pTableMetaInfo->vgroupIndex; assert(index >= 0); - + + SVgroupInfo* pVgroupInfo = NULL; if (pTableMetaInfo->vgroupList->numOfVgroups > 0) { assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; } + + vgId = pVgroupInfo->vgId; + tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); } else { - pVgroupInfo = &pTableMeta->vgroupInfo; + vgId = pTableMeta->vgId; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); } - assert(pVgroupInfo != NULL); + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - tscSetDnodeEpSet(pSql, pVgroupInfo); - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); + pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->id.tid); @@ -596,7 +665,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo); + tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); @@ -613,17 +682,18 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char pMsg += sizeof(STableIdInfo); } } - - tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, - pTableMeta->id.tid, pTableMeta->id.uid); - + + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); return pMsg; } int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - int32_t size = tscEstimateQueryMsgSize(pCmd, pCmd->clauseIndex); + int32_t size = tscEstimateQueryMsgSize(pSql, pCmd->clauseIndex); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); @@ -656,7 +726,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList); - + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); @@ -679,10 +750,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->tableLimit = htobe64(pQueryInfo->tableLimit); - + pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); + pQueryMsg->sqlstrLen = htonl(sqlLen); + pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); + size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number @@ -695,12 +769,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; - if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || - pColSchema->type > TSDB_DATA_TYPE_NCHAR) { + if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || !isValidDataType(pColSchema->type)) { + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscError("%p tid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", - pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, + pSql, pTableMeta->id.tid, pTableMeta->id.uid, n, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, pColSchema->name); - return TSDB_CODE_TSC_INVALID_SQL; } @@ -741,6 +817,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("%p table has already been destroyed", pSql); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { tscError("%p table schema is not matched with parsed sql", pSql); return TSDB_CODE_TSC_INVALID_SQL; @@ -766,7 +849,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen); pMsg += pExpr->param[j].nLen; } else { - pSqlFuncExpr->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64Key); + pSqlFuncExpr->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64); } } @@ -784,6 +867,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i); SSqlExpr *pExpr = pField->pSqlExpr; if (pExpr != NULL) { + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("%p table has already been destroyed", pSql); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { tscError("%p table schema is not matched with parsed sql", pSql); return TSDB_CODE_TSC_INVALID_SQL; @@ -806,7 +896,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen); pMsg += pExpr->param[j].nLen; } else { - pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64Key); + pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64); } } @@ -851,13 +941,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); - *((int16_t *)pMsg) = pCol->colId; + *((int16_t *)pMsg) = htons(pCol->colId); pMsg += sizeof(pCol->colId); - *((int16_t *)pMsg) += pCol->colIndex; + *((int16_t *)pMsg) += htons(pCol->colIndex); pMsg += sizeof(pCol->colIndex); - *((int16_t *)pMsg) += pCol->flag; + *((int16_t *)pMsg) += htons(pCol->flag); pMsg += sizeof(pCol->flag); memcpy(pMsg, pCol->name, tListLen(pCol->name)); @@ -884,10 +974,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) || - (pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) { + (!isValidDataType(pColSchema->type))) { + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + tscError("%p tid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", - pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, total, numOfTagColumns, - pCol->colIndex.columnIndex, pColSchema->name); + pSql, pTableMeta->id.tid, pTableMeta->id.uid, n, total, numOfTagColumns, pCol->colIndex.columnIndex, pColSchema->name); return TSDB_CODE_TSC_INVALID_SQL; } @@ -915,13 +1007,16 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += pCond->len; } } - - if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { - *pMsg = 0; - pMsg++; - } else { - strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); - pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + + if (pQueryInfo->bufLen > 0) { + memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); + pMsg += pQueryInfo->bufLen; + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; } // compressed ts block @@ -942,6 +1037,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); } + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); tscDebug("%p msg built success, len:%d bytes", pSql, msgLen); @@ -963,7 +1061,8 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { assert(pCmd->numOfClause == 1); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pCreateDbMsg->db, pTableMetaInfo->name, sizeof(pCreateDbMsg->db)); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); + assert(code == TSDB_CODE_SUCCESS); return TSDB_CODE_SUCCESS; } @@ -977,7 +1076,9 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; - strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); + + SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); + strncpy(pCreate->ep, t0->z, t0->n); pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE; @@ -994,13 +1095,13 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; - SStrToken *pName = &pInfo->pDCLInfo->user.user; - SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken *pName = &pInfo->pMiscInfo->user.user; + SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; strncpy(pAlterMsg->user, pName->z, pName->n); strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); - SCreateAcctSQL *pAcctOpt = &pInfo->pDCLInfo->acctOpt; + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); @@ -1040,7 +1141,7 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; - SUserInfo *pUser = &pInfo->pDCLInfo->user; + SUserInfo *pUser = &pInfo->pMiscInfo->user; strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); pAlterMsg->flag = (int8_t)pUser->type; @@ -1080,8 +1181,11 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db)); - pDropDbMsg->ignoreNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0; + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); + assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); + + pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DB; return TSDB_CODE_SUCCESS; @@ -1098,15 +1202,19 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - strcpy(pDropTableMsg->tableId, pTableMetaInfo->name); - pDropTableMsg->igNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0; + tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); + pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_TABLE; return TSDB_CODE_SUCCESS; } int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; + + char dnodeEp[TSDB_EP_LEN] = {0}; + tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); + pCmd->payloadLen = sizeof(SDropDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); @@ -1114,43 +1222,28 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep)); + tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE; return TSDB_CODE_SUCCESS; } -int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SDropUserMsg); - pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); + char user[TSDB_USER_LEN] = {0}; + tstrncpy(user, pCmd->payload, TSDB_USER_LEN); - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SDropUserMsg); - pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT; + pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_CM_DROP_USER:TSDB_MSG_TYPE_CM_DROP_ACCT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); + SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; + tstrncpy(pDropMsg->user, user, tListLen(user)); return TSDB_CODE_SUCCESS; } @@ -1166,7 +1259,7 @@ int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - strcpy(pUseDbMsg->db, pTableMetaInfo->name); + tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB; return TSDB_CODE_SUCCESS; @@ -1186,14 +1279,16 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - size_t nameLen = strlen(pTableMetaInfo->name); - if (nameLen > 0) { - tstrncpy(pShowMsg->db, pTableMetaInfo->name, sizeof(pShowMsg->db)); // prefix is set here + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + pthread_mutex_unlock(&pObj->mutex); } else { - tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); } - SShowInfo *pShowInfo = &pInfo->pDCLInfo->showOpt; + SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { @@ -1210,7 +1305,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pEpAddr->n); } - pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen; + pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); return TSDB_CODE_SUCCESS; } @@ -1234,12 +1329,12 @@ int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &(pSql->cmd); - - int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg); + int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo; if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { - size += sizeof(STagData); + int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); + size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); } else { size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); } @@ -1252,12 +1347,12 @@ int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - int msgLen = 0; - SSchema * pSchema; - int size = 0; + int msgLen = 0; + int size = 0; + SSchema *pSchema; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // Reallocate the payload size @@ -1267,33 +1362,52 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; - strcpy(pCreateTableMsg->tableId, pTableMetaInfo->name); - // use dbinfo from table id without modifying current db info - tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateTableMsg->db); + SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); + char* pMsg = NULL; - SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo; + int8_t type = pInfo->pCreateTableInfo->type; + if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value + SArray* list = pInfo->pCreateTableInfo->childTableInfo; + + int32_t numOfTables = (int32_t) taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); - pCreateTableMsg->igExists = pCreateTable->existCheck ? 1 : 0; - pCreateTableMsg->numOfColumns = htons(pCmd->numOfCols); - pCreateTableMsg->numOfTags = htons(pCmd->count); + pMsg = (char*) pCreateMsg; + for(int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; - pCreateTableMsg->sqlLen = 0; - char *pMsg = (char *)pCreateTableMsg->schema; + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); - int8_t type = pInfo->pCreateTableInfo->type; - if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value - STagData* pTag = &pInfo->pCreateTableInfo->usingInfo.tagdata; - *(int32_t*)pMsg = htonl(pTag->dataLen); - pMsg += sizeof(int32_t); - memcpy(pMsg, pTag->name, sizeof(pTag->name)); - pMsg += sizeof(pTag->name); - memcpy(pMsg, pTag->data, pTag->dataLen); - pMsg += pTag->dataLen; + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableName, p->fullname); + pCreate->igExists = (p->igExist)? 1 : 0; + + // use dbinfo from table id without modifying current db info + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*) pCreate); + pCreate->len = htonl(len); + } } else { // create (super) table - pSchema = (SSchema *)pCreateTableMsg->schema; + pCreateTableMsg->numOfTables = htonl(1); // only one table will be created + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); + assert(code == 0); + + SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo; + + pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; + pCreateMsg->numOfColumns = htons(pCmd->numOfCols); + pCreateMsg->numOfTags = htons(pCmd->count); + + pCreateMsg->sqlLen = 0; + pMsg = (char *)pCreateMsg->schema; + + pSchema = (SSchema *)pCreateMsg->schema; for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); @@ -1310,7 +1424,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQuerySQL *pQuerySql = pInfo->pCreateTableInfo->pSelect; strncpy(pMsg, pQuerySql->selectToken.z, pQuerySql->selectToken.n + 1); - pCreateTableMsg->sqlLen = htons(pQuerySql->selectToken.n + 1); + pCreateMsg->sqlLen = htons(pQuerySql->selectToken.n + 1); pMsg += pQuerySql->selectToken.n + 1; } } @@ -1340,7 +1454,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SAlterTableSQL *pAlterInfo = pInfo->pAlterInfo; + SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for alter table msg", pSql); @@ -1348,9 +1462,8 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; - tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db); - strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); + tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); pAlterTableMsg->type = htons(pAlterInfo->type); pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); @@ -1386,10 +1499,14 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.vgId > 0); - tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epSet); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); return TSDB_CODE_SUCCESS; } @@ -1401,7 +1518,7 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db)); + tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); return TSDB_CODE_SUCCESS; } @@ -1455,7 +1572,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { if (code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -1484,7 +1601,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { int32_t code = pRes->code; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; } @@ -1503,7 +1620,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; @@ -1526,9 +1643,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // TODO refactor full_name char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); db = strstr(pObj->db, TS_PATH_DELIMITER); + db = (db == NULL) ? pObj->db : db + 1; tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); @@ -1539,24 +1661,23 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd * pCmd = &pSql->cmd; + SSqlCmd *pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload; + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pInfoMsg->tableFname); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } - STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload; - strcpy(pInfoMsg->tableId, pTableMetaInfo->name); pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0); char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - if (pCmd->autoCreated && pCmd->pTagData != NULL) { - int len = htonl(pCmd->pTagData->dataLen); - if (len > 0) { - len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen); - memcpy(pInfoMsg->tags, pCmd->pTagData, len); - pMsg += len; - } + if (pCmd->autoCreated && pCmd->tagData.dataLen != 0) { + pMsg = serializeTagData(&pCmd->tagData, pMsg); } pCmd->payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); @@ -1607,33 +1728,6 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return 0; } -//static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { -//// const int32_t defaultSize = -//// minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS; -//// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); -//// -//// int32_t n = 0; -//// size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); -//// for (int32_t i = 0; i < size; ++i) { -//// assert(0); -////// n += strlen(pQueryInfo->tagCond.cond[i].cond); -//// } -//// -//// int32_t tagLen = n * TSDB_NCHAR_SIZE; -//// if (pQueryInfo->tagCond.tbnameCond.cond != NULL) { -//// tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; -//// } -//// -//// int32_t joinCondLen = (TSDB_TABLE_FNAME_LEN + sizeof(int16_t)) * 2; -//// int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; -//// -//// int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); -//// -//// int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; -//// -//// return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); -//} - int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; @@ -1646,9 +1740,10 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); - size_t size = sizeof(pTableMetaInfo->name); - tstrncpy(pMsg, pTableMetaInfo->name, size); - pMsg += size; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); + assert(code == TSDB_CODE_SUCCESS); + + pMsg += TSDB_TABLE_FNAME_LEN; } pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; @@ -1715,11 +1810,11 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->uid = htobe64(pMetaMsg->uid); pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->tid, pMetaMsg->tableId); + pMetaMsg->tid, pMetaMsg->tableFname); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1748,35 +1843,74 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(i == 0); } - assert(pSchema->type >= TSDB_DATA_TYPE_BOOL && pSchema->type <= TSDB_DATA_TYPE_NCHAR); pSchema++; } - - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - // todo add one more function: taosAddDataIfNotExists(); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("%p invalid table meta from mnode, name:%s", pSql, tNameGetTableName(&pTableMetaInfo->name)); + return TSDB_CODE_TSC_INVALID_VALUE; + } - if (pTableMetaInfo->pTableMeta == NULL) { - free(pTableMeta); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // check if super table hashmap or not + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + // super tableMeta data alreay exists, create it according to tableMeta and add it to hash map + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashPut(tscTableMetaInfo, name, strlen(name), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashPut(tscTableMetaInfo, name, strlen(name), pTableMeta, s); } - tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name); + // update the vgroupInfo if needed + if (pTableMeta->vgId > 0) { + int32_t vgId = pTableMeta->vgId; + assert(pTableMeta->tableType != TSDB_SUPER_TABLE); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, &pMetaMsg->vgroup)) || + (vgroupInfo.inUse < 0)) { // vgroup info exists, compare with it + vgroupInfo = createNewVgroupInfo(&pMetaMsg->vgroup); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add new VgroupInfo, vgId:%d, total:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } + } + + tscDebug("%p recv table meta, uid:%" PRIu64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, + tNameGetTableName(&pTableMetaInfo->name)); + free(pTableMeta); - return TSDB_CODE_SUCCESS; } /** * multi table meta rsp pkg format: - * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 - * |...... 1B 1B 4B + * | STaosRsp | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * |...... 1B 4B **/ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { #if 0 @@ -1868,9 +2002,6 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { // // rsp += tagLen; // int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with STableMeta in cache - // - // pMeta->index = 0; - // (void)taosCachePut(tscMetaCache, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer); // } } @@ -1883,16 +2014,20 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { } int tscProcessSTableVgroupRsp(SSqlObj *pSql) { + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + SSqlRes* pRes = &pSql->res; // NOTE: the order of several table must be preserved. SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); - - // master sqlObj locates in param - SSqlObj* parent = pSql->param; - assert(parent != NULL); SSqlCmd* pCmd = &parent->cmd; for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { @@ -1926,18 +2061,16 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { pMsg += size; } + + taosReleaseRef(tscObjRef, parent->self); return pSql->res.code; } -/* - * current process do not use the cache at all - */ int tscProcessShowRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg; SShowRsp * pShow; SSchema * pSchema; - char key[20]; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -1962,20 +2095,10 @@ int tscProcessShowRsp(SSqlObj *pSql) { pSchema++; } - key[0] = pCmd->msgType + 'a'; - strcpy(key + 1, "showlist"); + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false); - } - - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - - pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } @@ -1998,13 +2121,10 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); - - tfree(pTableMeta); return 0; } -// TODO multithread problem -static void createHBObj(STscObj* pObj) { +static void createHbObj(STscObj* pObj) { if (pObj->hbrid != 0) { return; } @@ -2048,14 +2168,21 @@ int tscProcessConnectRsp(SSqlObj *pSql) { SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response + + pthread_mutex_lock(&pObj->mutex); int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); + pthread_mutex_unlock(&pObj->mutex); if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); tscUpdateMgmtEpSet(pSql, &pConnect->epSet); + + for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { + tscDebug("%p epSet.fqdn[%d]: %s, pObj:%p", pSql, i, pConnect->epSet.fqdn[i], pObj); + } } strcpy(pObj->sversion, pConnect->serverVersion); @@ -2063,7 +2190,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { pObj->superAuth = pConnect->superAuth; pObj->connId = htonl(pConnect->connId); - createHBObj(pObj); + createHbObj(pObj); //launch a timer to send heartbeat to maintain the connection and send status to mnode taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer); @@ -2074,61 +2201,50 @@ int tscProcessConnectRsp(SSqlObj *pSql) { int tscProcessUseDbRsp(SSqlObj *pSql) { STscObj * pObj = pSql->pTscObj; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - - tstrncpy(pObj->db, pTableMetaInfo->name, sizeof(pObj->db)); - return 0; + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; } int tscProcessDropDbRsp(SSqlObj *pSql) { - pSql->pTscObj->db[0] = 0; - taosCacheEmpty(tscMetaCache); + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + + taosHashEmpty(tscTableMetaInfo); return 0; } int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } - - /* - * 1. if a user drops one table, which is the only table in a vnode, remove operation will incur vnode to be removed. - * 2. Then, a user creates a new metric followed by a table with identical name of removed table but different schema, - * here the table will reside in a new vnode. - * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache - * instead. - */ - tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); + //The cached tableMeta is expired in this case, so clean it in hash table + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); - if (pTableMetaInfo->pTableMeta) { - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); - } + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tscDebug("%p remove table meta after drop table:%s, numOfRemain:%d", pSql, name, (int32_t) taosHashGetSize(tscTableMetaInfo)); + pTableMetaInfo->pTableMeta = NULL; return 0; } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); - tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); + tscDebug("%p remove tableMeta in hashMap after alter-table: %s", pSql, name); - if (pTableMetaInfo->pTableMeta) { - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); - if (isSuperTable) { // if it is a super table, reset whole query cache - tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); - taosCacheEmpty(tscMetaCache); - } + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + taosHashEmpty(tscTableMetaInfo); } return 0; @@ -2138,6 +2254,7 @@ int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { UNUSED(pSql); return 0; } + int tscProcessShowCreateRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 1); } @@ -2158,6 +2275,8 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; if (pRetrieve == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -2213,7 +2332,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { +static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("%p malloc failed for new sqlobj to get table meta", pSql); @@ -2238,61 +2357,76 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); + tNameAssign(&pNewMeterMetaInfo->name, &pTableMetaInfo->name); - if (pSql->cmd.pTagData != NULL) { - int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); - pNew->cmd.pTagData = calloc(1, size); - if (pNew->cmd.pTagData == NULL) { + if (pSql->cmd.autoCreated) { + int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); + if (code != TSDB_CODE_SUCCESS) { tscError("%p malloc failed for new tag data to get table meta", pSql); tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); } tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); + registerSqlObj(pNew); + pNew->fp = tscTableMetaCallBack; - pNew->param = pSql; + pNew->param = (void *)pSql->self; - registerSqlObj(pNew); + tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated } return code; } int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { - assert(strlen(pTableMetaInfo->name) != 0); + assert(tIsValidName(&pTableMetaInfo->name)); - // If this STableMetaInfo owns a table meta, release it first - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false); - } - - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMetaInfo->pTableMeta != NULL) { - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, - tinfo.numOfTags, pTableMetaInfo->pTableMeta); + tfree(pTableMetaInfo->pTableMeta); + + uint32_t size = tscGetTableMetaMaxSize(); + pTableMetaInfo->pTableMeta = calloc(1, size); + + pTableMetaInfo->pTableMeta->tableType = -1; + pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + size_t len = strlen(name); + taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMetaInfo->pTableMeta, -1); + + // TODO resize the tableMeta + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + if (pMeta->id.uid > 0) { + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo); + } + } return TSDB_CODE_SUCCESS; } - - return getTableMetaFromMgmt(pSql, pTableMetaInfo); + + return getTableMetaFromMnode(pSql, pTableMetaInfo); } -int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { +int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { pSql->cmd.autoCreated = createIfNotExists; return tscGetTableMeta(pSql, pTableMetaInfo); } /** - * retrieve table meta from mnode, and update the local table meta cache. + * retrieve table meta from mnode, and update the local table meta hashmap. * @param pSql sql object * @param tableIndex table index * @return status code @@ -2300,17 +2434,27 @@ int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + char name[TSDB_TABLE_FNAME_LEN] = {0}; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); + if (code != TSDB_CODE_SUCCESS) { + tscError("%p failed to generate the table full name", pSql); + return TSDB_CODE_TSC_INVALID_SQL; + } + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pTableMeta) { - tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta); + if (pTableMeta) { + tscDebug("%p update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64, pSql, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); } - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); - return getTableMetaFromMgmt(pSql, pTableMetaInfo); + // remove stored tableMeta info in hash table + size_t len = strlen(name); + taosHashRemove(tscTableMetaInfo, name, len); + + return getTableMetaFromMnode(pSql, pTableMetaInfo); } static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) { @@ -2350,8 +2494,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); + STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -2362,10 +2506,15 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; registerSqlObj(pNew); + tscDebug("%p svgroupRid from %" PRId64 " to %" PRId64 , pSql, pSql->svgroupRid, pNew->self); + + pSql->svgroupRid = pNew->self; + + tscDebug("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables); pNew->fp = tscTableMetaCallBack; - pNew->param = pSql; + pNew->param = (void *)pSql->self; code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; @@ -2386,8 +2535,8 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; - tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserMsg; - tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index fae5b5856fb8804ac9d8d9d67eb1592db16bdaff..13539a9b197875210d76c86bd93ec986190c0c37 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -15,7 +15,7 @@ #include "hash.h" #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "tkey.h" #include "tcache.h" #include "tnote.h" @@ -47,12 +47,14 @@ static bool validUserName(const char* user) { } static bool validPassword(const char* passwd) { - return validImpl(passwd, TSDB_PASSWORD_LEN - 1); + return validImpl(passwd, TSDB_KEY_LEN - 1); } static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { - taos_init(); + if (taos_init()) { + return NULL; + } if (!validUserName(user)) { terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; @@ -90,9 +92,11 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa } else { if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL; } + char rpcKey[512] = {0}; + snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, pass, ip, port); - void *pDnodeConn = NULL; - if (tscInitRpc(user, secretEncrypt, &pDnodeConn) != 0) { + void *pRpcObj = NULL; + if (tscAcquireRpc(rpcKey, user, secretEncrypt, &pRpcObj) != 0) { terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return NULL; } @@ -100,24 +104,21 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); + tscReleaseRpc(pRpcObj); return NULL; } - // set up tscObj's mgmtEpSet - pObj->tscCorMgmtEpSet = (SRpcCorEpSet *)malloc(sizeof(SRpcCorEpSet)); - if (NULL == pObj->tscCorMgmtEpSet) { + + pObj->tscCorMgmtEpSet = malloc(sizeof(SRpcCorEpSet)); + if (pObj->tscCorMgmtEpSet == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); - free(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pRpcObj); free(pObj); + return NULL; } - memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(SRpcCorEpSet)); - - pObj->signature = pObj; - pObj->pDnodeConn = pDnodeConn; - T_REF_INIT_VAL(pObj, 1); + memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(corMgmtEpSet)); - + pObj->signature = pObj; + pObj->pRpcObj = (SRpcObj *)pRpcObj; tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); memcpy(pObj->pass, secretEncrypt, secretEncryptLen); @@ -127,8 +128,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa /* db name is too long */ if (len >= TSDB_DB_NAME_LEN) { terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; - rpcClose(pDnodeConn); - free(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pRpcObj); free(pObj); return NULL; } @@ -145,8 +145,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (NULL == pSql) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); - free(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pRpcObj); free(pObj); return NULL; } @@ -162,9 +161,8 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); + tscReleaseRpc(pRpcObj); free(pSql); - free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } @@ -172,11 +170,9 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (taos != NULL) { *taos = pObj; } - + pObj->rid = taosAddRef(tscRefId, pObj); registerSqlObj(pSql); - tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - pObj->rid = taosAddRef(tscRefId, pObj); return pSql; } @@ -205,7 +201,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, return NULL; } - tscDebug("%p DB connection is opening, dnodeConn:%p", pObj, pObj->pDnodeConn); + tscDebug("%p DB connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj, pObj->pRpcObj->pDnodeConn); taos_free_result(pSql); // version compare only requires the first 3 segments of the version string @@ -242,11 +238,11 @@ TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t us uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port) { char ipBuf[TSDB_EP_LEN] = {0}; char userBuf[TSDB_USER_LEN] = {0}; - char passBuf[TSDB_PASSWORD_LEN] = {0}; + char passBuf[TSDB_KEY_LEN] = {0}; char dbBuf[TSDB_DB_NAME_LEN] = {0}; strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen)); strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passBuf, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(passBuf, pass, MIN(TSDB_KEY_LEN - 1, passLen)); strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); } @@ -282,40 +278,31 @@ void taos_close(TAOS *taos) { return; } - tscDebug("%p try to free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); + tscDebug("%p try to free tscObj", pObj); if (pObj->signature != pObj) { tscDebug("%p already closed or invalid tscObj", pObj); return; } - // make sure that the close connection can only be executed once. - pObj->signature = NULL; - taosTmrStopA(&(pObj->pTimer)); - - if (pObj->hbrid > 0) { + if (RID_VALID(pObj->hbrid)) { SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); if (pHb != NULL) { - if (pHb->rpcRid > 0) { // wait for rsp from dnode + if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode rpcCancelRequest(pHb->rpcRid); pHb->rpcRid = -1; } tscDebug("%p HB is freed", pHb); - taos_free_result(pHb); taosReleaseRef(tscObjRef, pHb->self); +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pHb->rspSem, 0, 0); +#endif // __APPLE__ + taos_free_result(pHb); } } - int32_t ref = T_REF_DEC(pObj); - assert(ref >= 0); - - if (ref > 0) { - tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn); - return; - } - - tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); - + tscDebug("%p all sqlObj are freed, free tscObj", pObj); taosRemoveRef(tscRefId, pObj->rid); } @@ -331,7 +318,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { tsem_post(&pSql->rspSem); } -TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) { +TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, int64_t* res) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { terrno = TSDB_CODE_TSC_DISCONNECTED; @@ -344,7 +331,7 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES return NULL; } - taosNotePrintTsc(sqlstr); + nPrintTsc("%s", sqlstr); SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { @@ -357,7 +344,7 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); if (res != NULL) { - *res = pSql; + atomic_store_64(res, pSql->self); } tsem_wait(&pSql->rspSem); @@ -368,7 +355,7 @@ TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL); } -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) { +TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res) { return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res); } @@ -458,24 +445,6 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pFieldInfo->final; } -int taos_retrieve(TAOS_RES *res) { - if (res == NULL) return 0; - SSqlObj *pSql = (SSqlObj *)res; - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - if (pSql == NULL || pSql->signature != pSql) return 0; - if (pRes->qhandle == 0) return 0; - - tscResetForNextRetrieve(pRes); - - if (pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); - return pRes->numOfRows; -} - static bool needToFetchNewBlock(SSqlObj* pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -711,6 +680,8 @@ static void tscKillSTableQuery(SSqlObj *pSql) { // set the master sqlObj flag to cancel query pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + tscLockByThread(&pSql->squeryLock); + for (int i = 0; i < pSql->subState.numOfSub; ++i) { // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; @@ -726,10 +697,16 @@ static void tscKillSTableQuery(SSqlObj *pSql) { pSubObj->rpcRid = -1; } - tscQueueAsyncRes(pSubObj); - taosReleaseRef(tscObjRef, pSubObj->self); + tscAsyncResultOnError(pSubObj); + // taosRelekaseRef(tscObjRef, pSubObj->self); + } + + if (pSql->subState.numOfSub <= 0) { + tscAsyncResultOnError(pSql); } + tscUnlockByThread(&pSql->squeryLock); + tscDebug("%p super table query cancelled", pSql); } @@ -762,7 +739,7 @@ void taos_stop_query(TAOS_RES *res) { pSql->rpcRid = -1; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -790,6 +767,7 @@ bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { int len = 0; + for (int i = 0; i < num_fields; ++i) { if (i > 0) { str[len++] = ' '; @@ -805,18 +783,34 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) len += sprintf(str + len, "%d", *((int8_t *)row[i])); break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: len += sprintf(str + len, "%d", *((int16_t *)row[i])); break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + case TSDB_DATA_TYPE_INT: len += sprintf(str + len, "%d", *((int32_t *)row[i])); break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: { float fv = 0; fv = GET_FLOAT_VAL(row[i]); @@ -831,13 +825,15 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { - size_t xlen = 0; - for (xlen = 0; xlen < fields[i].bytes - VARSTR_HEADER_SIZE; xlen++) { - char c = ((char *)row[i])[xlen]; - if (c == 0) break; - str[len++] = c; + int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); + if (fields[i].type == TSDB_DATA_TYPE_BINARY) { + assert(charLen <= fields[i].bytes); + } else { + assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE); } - str[len] = 0; + + memcpy(str + len, row[i], charLen); + len += charLen; } break; case TSDB_DATA_TYPE_TIMESTAMP: @@ -900,9 +896,9 @@ int taos_validate_sql(TAOS *taos, const char *sql) { strtolower(pSql->sqlstr, sql); pCmd->curSql = NULL; - if (NULL != pCmd->pTableList) { - taosHashCleanup(pCmd->pTableList); - pCmd->pTableList = NULL; + if (NULL != pCmd->pTableBlockHashList) { + taosHashCleanup(pCmd->pTableBlockHashList); + pCmd->pTableBlockHashList = NULL; } pSql->fp = asyncCallback; @@ -926,7 +922,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) { // must before clean the sqlcmd object - tscResetSqlCmdObj(&pSql->cmd, false); + tscResetSqlCmd(&pSql->cmd, false); SSqlCmd *pCmd = &pSql->cmd; @@ -979,13 +975,14 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if (++pCmd->count > TSDB_MULTI_METERMETA_MAX_NUM) { + if (++pCmd->count > TSDB_MULTI_TABLEMETA_MAX_NUM) { code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; sprintf(pCmd->payload, "tables over the max number"); return code; } - if (payloadLen + strlen(pTableMetaInfo->name) + 128 >= pCmd->allocSize) { + int32_t xlen = tNameLen(&pTableMetaInfo->name); + if (payloadLen + xlen + 128 >= pCmd->allocSize) { char *pNewMem = realloc(pCmd->payload, pCmd->allocSize + tblListLen); if (pNewMem == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -998,7 +995,9 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t pMsg = pCmd->payload; } - payloadLen += sprintf(pMsg + payloadLen, "%s,", pTableMetaInfo->name); + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + payloadLen += sprintf(pMsg + payloadLen, "%s,", n); } *(pMsg + payloadLen) = '\0'; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 68c3bcae165050863cc4bf9c92a1510581531c3a..a9cd1965e8b3e6282aaf7601011e78e5ff4bc7bb 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -65,44 +65,67 @@ static int64_t tscGetRetryDelayTime(SSqlStream* pStream, int64_t slidingTime, in return retryDelta; } -static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { - SSqlStream *pStream = (SSqlStream *)pMsg->ahandle; - SSqlObj * pSql = pStream->pSql; +static void setRetryInfo(SSqlStream* pStream, int32_t code) { + SSqlObj* pSql = pStream->pSql; - pSql->fp = tscProcessStreamQueryCallback; - pSql->fetchFp = tscProcessStreamQueryCallback; - pSql->param = pStream; + pSql->res.code = code; + int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); + tscDebug("%p stream:%p, get table Meta failed, retry in %" PRId64 "ms", pSql, pStream, retryDelayTime); + tscSetRetryTimer(pStream, pSql, retryDelayTime); +} + +static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) { + SSqlStream *pStream = (SSqlStream *)param; + assert(pStream->pSql == tres); + + SSqlObj* pSql = (SSqlObj*) tres; + + pSql->fp = doLaunchQuery; + pSql->fetchFp = doLaunchQuery; pSql->res.completed = false; - + + if (code != TSDB_CODE_SUCCESS) { + setRetryInfo(pStream, code); + return; + } + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int code = tscGetTableMeta(pSql, pTableMetaInfo); - pSql->res.code = code; - + code = tscGetTableMeta(pSql, pTableMetaInfo); if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscGetSTableVgroupInfo(pSql, 0); - pSql->res.code = code; } - // failed to get meter/metric meta, retry in 10sec. - if (code != TSDB_CODE_SUCCESS) { - int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscDebug("%p stream:%p,get metermeta failed, retry in %" PRId64 "ms", pStream->pSql, pStream, retryDelayTime); - tscSetRetryTimer(pStream, pSql, retryDelayTime); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } - } else { - tscTansformSQLFuncForSTableQuery(pQueryInfo); - tscDebug("%p stream:%p start stream query on:%s", pSql, pStream, pTableMetaInfo->name); - tscDoQuery(pStream->pSql); + // failed to get table Meta or vgroup list, retry in 10sec. + if (code == TSDB_CODE_SUCCESS) { + tscTansformFuncForSTableQuery(pQueryInfo); + tscDebug("%p stream:%p, start stream query on:%s", pSql, pStream, tNameGetTableName(&pTableMetaInfo->name)); + + pSql->fp = tscProcessStreamQueryCallback; + pSql->fetchFp = tscProcessStreamQueryCallback; + tscDoQuery(pSql); tscIncStreamExecutionCount(pStream); + } else { + setRetryInfo(pStream, code); } } +static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { + SSqlStream *pStream = (SSqlStream *)pMsg->ahandle; + doLaunchQuery(pStream, pStream->pSql, 0); +} + static void tscProcessStreamTimer(void *handle, void *tmrId) { SSqlStream *pStream = (SSqlStream *)handle; - if (pStream == NULL) return; - if (pStream->pTimer != tmrId) return; + if (pStream == NULL || pStream->pTimer != tmrId) { + return; + } + pStream->pTimer = NULL; pStream->numOfRes = 0; // reset the numOfRes. @@ -167,7 +190,11 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); @@ -266,12 +293,11 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf pStream->stime += 1; } - tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name, + tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->numOfRes); - // release the metric/meter meta information reference, so data in cache can be updated + tfree(pTableMetaInfo->pTableMeta); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; @@ -391,11 +417,16 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { tscSetRetryTimer(pStream, pSql, timer); } -static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { +static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { int64_t minIntervalTime = (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinIntervalTime * 1000L : tsMinIntervalTime; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + if (!pStream->isProject && pQueryInfo->interval.interval == 0) { + sprintf(pSql->cmd.payload, "the interval value is 0"); + return -1; + } if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) { tscWarn("%p stream:%p, original sample interval:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream, @@ -435,6 +466,8 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { pQueryInfo->interval.interval = 0; // clear the interval value to avoid the force time window split by query processor pQueryInfo->interval.sliding = 0; } + + return TSDB_CODE_SUCCESS; } static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) { @@ -484,34 +517,19 @@ static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) { return (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? timer / 1000L : timer; } -static void setErrorInfo(SSqlObj* pSql, int32_t code, char* info) { - if (pSql == NULL) { - return; - } - - SSqlCmd* pCmd = &pSql->cmd; - - pSql->res.code = code; - - if (info != NULL) { - strncpy(pCmd->payload, info, pCmd->payloadLen); - } -} - static void tscCreateStream(void *param, TAOS_RES *res, int code) { SSqlStream* pStream = (SSqlStream*)param; SSqlObj* pSql = pStream->pSql; SSqlCmd* pCmd = &pSql->cmd; if (code != TSDB_CODE_SUCCESS) { - setErrorInfo(pSql, code, pCmd->payload); - tscError("%p open stream failed, sql:%s, reason:%s, code:0x%08x", pSql, pSql->sqlstr, pCmd->payload, code); + pSql->res.code = code; + tscError("%p open stream failed, sql:%s, reason:%s, code:%s", pSql, pSql->sqlstr, pCmd->payload, tstrerror(code)); + pStream->fp(pStream->param, NULL, NULL); return; } - registerSqlObj(pSql); - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -522,17 +540,29 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { pStream->ctime = taosGetTimestamp(pStream->precision); pStream->etime = pQueryInfo->window.ekey; - tscAddIntoStreamList(pStream); + if (tscSetSlidingWindowInfo(pSql, pStream) != TSDB_CODE_SUCCESS) { + pSql->res.code = code; + + tscError("%p stream %p open failed, since the interval value is incorrect", pSql, pStream); + pStream->fp(pStream->param, NULL, NULL); + return; + } - tscSetSlidingWindowInfo(pSql, pStream); pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, pStream->stime); int64_t starttime = tscGetLaunchTimestamp(pStream); pCmd->command = TSDB_SQL_SELECT; + + tscAddIntoStreamList(pStream); + taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer); tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql, - pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); + pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); +} + +void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) { + pStream->dstTable = dstTable; } TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), @@ -581,12 +611,15 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p pSql->fp = tscCreateStream; pSql->fetchFp = tscCreateStream; + + registerSqlObj(pSql); + int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_SUCCESS) { tscCreateStream(pStream, pSql, code); } else if (code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { tscError("%p open stream failed, sql:%s, code:%s", pSql, sqlstr, tstrerror(pRes->code)); - tscFreeSqlObj(pSql); + taosReleaseRef(tscObjRef, pSql->self); free(pStream); return NULL; } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 52b74f7502db139f18d8230eefb36d41bae4079f..f3d7ef28c05fc3128622f8217341d52327d79ec0 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { SSub* pSub = (SSub*)sub; SSubscriptionProgress target = {.uid = uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); + SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); if (p == NULL) { return dflt; } @@ -69,14 +69,17 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { } void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { - if( sub == NULL) + if( sub == NULL) { return; + } + SSub* pSub = (SSub*)sub; SSubscriptionProgress target = {.uid = uid, .key = ts}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); + SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); if (p != NULL) { p->key = ts; + tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key); } } @@ -267,7 +270,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); + SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); if (p == NULL) { taosArrayClear(pSub->progress); taosArrayPush(pSub->progress, &target); @@ -310,7 +313,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) { char buf[TSDB_MAX_SQL_LEN]; sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(buf, "r"); + FILE* fp = fopen(buf, "rb"); if (fp == NULL) { tscDebug("subscription progress file does not exist: %s", pSub->topic); return 1; @@ -365,7 +368,7 @@ void tscSaveSubscriptionProgress(void* sub) { } sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(path, "w+"); + FILE* fp = fopen(path, "wb+"); if (fp == NULL) { tscError("failed to create progress file for subscription: %s", pSub->topic); return; @@ -502,6 +505,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key; + tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); } if (pSub->pTimer == NULL) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 819a323db5e947df46158b9665c0f5d5c491f532..78e9c6829081cb61f5fa8e2ed55397242ba357a4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -13,16 +13,17 @@ * along with this program. If not, see . */ #define _GNU_SOURCE - + #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "qTsbuf.h" #include "tcompare.h" #include "tscLog.h" #include "tscSubquery.h" #include "tschemautil.h" #include "tsclient.h" +#include "qUtil.h" typedef struct SInsertSupporter { SSqlObj* pSql; @@ -55,6 +56,68 @@ static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { } } +static void subquerySetState(SSqlObj *pSql, SSubqueryState *subState, int idx, int8_t state) { + assert(idx < subState->numOfSub); + assert(subState->states); + + pthread_mutex_lock(&subState->mutex); + + tscDebug("subquery:%p,%d state set to %d", pSql, idx, state); + + subState->states[idx] = state; + + pthread_mutex_unlock(&subState->mutex); +} + +static bool allSubqueryDone(SSqlObj *pParentSql) { + bool done = true; + SSubqueryState *subState = &pParentSql->subState; + + //lock in caller + + for (int i = 0; i < subState->numOfSub; i++) { + if (0 == subState->states[i]) { + tscDebug("%p subquery:%p,%d is NOT finished, total:%d", pParentSql, pParentSql->pSubs[i], i, subState->numOfSub); + done = false; + break; + } else { + tscDebug("%p subquery:%p,%d is finished, total:%d", pParentSql, pParentSql->pSubs[i], i, subState->numOfSub); + } + } + + return done; +} + +static bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) { + SSubqueryState *subState = &pParentSql->subState; + + assert(idx < subState->numOfSub); + + pthread_mutex_lock(&subState->mutex); + + bool done = allSubqueryDone(pParentSql); + + if (done) { + tscDebug("%p subquery:%p,%d all subs already done", pParentSql, pSql, idx); + + pthread_mutex_unlock(&subState->mutex); + + return false; + } + + tscDebug("%p subquery:%p,%d state set to 1", pParentSql, pSql, idx); + + subState->states[idx] = 1; + + done = allSubqueryDone(pParentSql); + + pthread_mutex_unlock(&subState->mutex); + + return done; +} + + + static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJoinSupporter* pSupporter2, STimeWindow * win) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); @@ -188,8 +251,10 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tsBufFlush(output2); tsBufDestroy(pSupporter1->pTSBuf); + pSupporter1->pTSBuf = NULL; tsBufDestroy(pSupporter2->pTSBuf); - + pSupporter2->pTSBuf = NULL; + TSKEY et = taosGetTimestampUs(); tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", @@ -219,12 +284,9 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) { assert (pSupporter->uid != 0); taosGetTmpfilePath("join-", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); - // todo handle error - if (pSupporter->f == NULL) { - tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); - } + // do NOT create file here to reduce crash generated file left issue + pSupporter->f = NULL; return pSupporter; } @@ -244,12 +306,19 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { tscFieldInfoClear(&pSupporter->fieldsInfo); + if (pSupporter->pTSBuf != NULL) { + tsBufDestroy(pSupporter->pTSBuf); + pSupporter->pTSBuf = NULL; + } + + unlink(pSupporter->path); + if (pSupporter->f != NULL) { fclose(pSupporter->f); - unlink(pSupporter->path); pSupporter->f = NULL; } + if (pSupporter->pVgroupTables != NULL) { taosArrayDestroy(pSupporter->pVgroupTables); pSupporter->pVgroupTables = NULL; @@ -361,10 +430,6 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // scan all subquery, if one sub query has only ts, ignore it tscDebug("%p start to launch secondary subqueries, %d out of %d needs to query", pSql, numOfSub, pSql->subState.numOfSub); - //the subqueries that do not actually launch the secondary query to virtual node is set as completed. - SSubqueryState* pState = &pSql->subState; - pState->numOfRemain = numOfSub; - bool success = true; for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { @@ -384,7 +449,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { } SQueryInfo *pSubQueryInfo = tscGetQueryInfoDetail(&pPrevSub->cmd, 0); - STSBuf *pTSBuf = pSubQueryInfo->tsBuf; + STSBuf *pTsBuf = pSubQueryInfo->tsBuf; pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj @@ -397,12 +462,13 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { success = false; break; } + tscClearSubqueryInfo(&pNew->cmd); pSql->pSubs[i] = pNew; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - pQueryInfo->tsBuf = pTSBuf; // transfer the ownership of timestamp comp-z data to the new created object + pQueryInfo->tsBuf = pTsBuf; // transfer the ownership of timestamp comp-z data to the new created object // set the second stage sub query for join process TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); @@ -446,7 +512,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS; - tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); tscPrintSelectClause(pNew, 0); tscFieldInfoUpdateOffset(pQueryInfo); @@ -459,7 +525,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); // set the tag column id for executor to extract correct tag value - pExpr->param[0] = (tVariant) {.i64Key = colId, .nType = TSDB_DATA_TYPE_BIGINT, .nLen = sizeof(int64_t)}; + pExpr->param[0] = (tVariant) {.i64 = colId, .nType = TSDB_DATA_TYPE_BIGINT, .nLen = sizeof(int64_t)}; pExpr->numOfParams = 1; } @@ -474,10 +540,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { } } + subquerySetState(pPrevSub, &pSql->subState, i, 0); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), - numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); } //prepare the subqueries object failed, abort @@ -511,21 +579,29 @@ void freeJoinSubqueryObj(SSqlObj* pSql) { SJoinSupporter* p = pSub->param; tscDestroyJoinSupporter(p); - if (pSub->res.code == TSDB_CODE_SUCCESS) { - taos_free_result(pSub); - } + taos_free_result(pSub); + pSql->pSubs[i] = NULL; } + if (pSql->subState.states) { + pthread_mutex_destroy(&pSql->subState.mutex); + } + + tfree(pSql->subState.states); + + pSql->subState.numOfSub = 0; } -static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { - assert(pSqlObj->subState.numOfRemain > 0); - - if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) { - tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code)); +static int32_t quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { + if (subAndCheckDone(pSqlSub, pSqlObj, pSupporter->subqueryIndex)) { + tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code)); freeJoinSubqueryObj(pSqlObj); + return 0; } + + return 1; + //tscDestroyJoinSupporter(pSupporter); } // update the query time range according to the join results on timestamp @@ -607,16 +683,20 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr prev = tt; } - pTableMetaInfo->pVgroupTables = result; pTableMetaInfo->vgroupIndex = 0; + + if (taosArrayGetSize(result) <= 0) { + pTableMetaInfo->pVgroupTables = NULL; + taosArrayDestroy(result); + } else { + pTableMetaInfo->pVgroupTables = result; - if (taosArrayGetSize(result) > 0) { SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1); tscDebug("%p vgId:%d, tables:%"PRIzu, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList)); } } -static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { +static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { SSqlCmd* pCmd = &pSql->cmd; tscClearSubqueryInfo(pCmd); tscFreeSqlResult(pSql); @@ -636,13 +716,13 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); // set the tags value for ts_comp function if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0); int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); - pExpr->param->i64Key = tagColId; + pExpr->param->i64 = tagColId; pExpr->numOfParams = 1; } @@ -666,7 +746,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* "%p subquery:%p tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, ts_comp query to retrieve timestamps, " "numOfExpr:%" PRIzu ", colList:%" PRIzu ", numOfOutputFields:%d, name:%s", pParent, pSql, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pQueryInfo->type, - tscSqlExprNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + tscSqlExprNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); tscProcessSql(pSql); } @@ -769,6 +849,17 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)); + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + + tscAsyncResultOnError(pParentSql); + + return; + } + // check for the error code firstly if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // todo retry if other subqueries are not failed @@ -777,9 +868,11 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); pParentSql->res.code = numOfRows; - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -794,9 +887,11 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p failed to malloc memory", pSql); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -836,16 +931,17 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // no data exists in next vnode, mark the query completed // only when there is no subquery exits any more, proceeds to get the intersect of the tuple sets. - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { + tscDebug("%p tagRetrieve:%p,%d completed, total:%d", pParentSql, tres, pSupporter->subqueryIndex, pParentSql->subState.numOfSub); return; - } + } SArray *s1 = NULL, *s2 = NULL; int32_t code = getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2); if (code != TSDB_CODE_SUCCESS) { freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); taosArrayDestroy(s1); taosArrayDestroy(s2); @@ -877,17 +973,19 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2); SSqlObj* psub1 = pParentSql->pSubs[0]; - ((SJoinSupporter*)psub1->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables); + ((SJoinSupporter*)psub1->param)->pVgroupTables = tscVgroupTableInfoDup(pTableMetaInfo1->pVgroupTables); SSqlObj* psub2 = pParentSql->pSubs[1]; - ((SJoinSupporter*)psub2->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables); + ((SJoinSupporter*)psub2->param)->pVgroupTables = tscVgroupTableInfoDup(pTableMetaInfo2->pVgroupTables); pParentSql->subState.numOfSub = 2; - pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; - + + memset(pParentSql->subState.states, 0, sizeof(pParentSql->subState.states[0]) * pParentSql->subState.numOfSub); + tscDebug("%p reset all sub states to 0", pParentSql); + for (int32_t m = 0; m < pParentSql->subState.numOfSub; ++m) { SSqlObj* sub = pParentSql->pSubs[m]; - issueTSCompQuery(sub, sub->param, pParentSql); + issueTsCompQuery(sub, sub->param, pParentSql); } } @@ -907,6 +1005,17 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert(!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)); + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + if (quitAllSubquery(pSql, pParentSql, pSupporter)){ + return; + } + + tscAsyncResultOnError(pParentSql); + + return; + } + // check for the error code firstly if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // todo retry if other subqueries are not failed yet @@ -914,13 +1023,33 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); pParentSql->res.code = numOfRows; - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)){ + return; + } - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } if (numOfRows > 0) { // write the compressed timestamp to disk file + if(pSupporter->f == NULL) { + pSupporter->f = fopen(pSupporter->path, "wb"); + + if (pSupporter->f == NULL) { + tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); + + pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); + + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + + tscAsyncResultOnError(pParentSql); + + return; + } + } + fwrite(pRes->data, (size_t)pRes->numOfRows, 1, pSupporter->f); fclose(pSupporter->f); pSupporter->f = NULL; @@ -930,7 +1059,12 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - tscQueueAsyncRes(pParentSql); + + if (quitAllSubquery(pSql, pParentSql, pSupporter)){ + return; + } + + tscAsyncResultOnError(pParentSql); return; } @@ -947,7 +1081,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // continue to retrieve ts-comp data from vnode if (!pRes->completed) { taosGetTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; taos_fetch_rows_a(tres, tsCompRetrieveCallback, param); @@ -973,7 +1107,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow taosGetTmpfilePath("ts-join", pSupporter->path); // TODO check for failure - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; // set the callback function @@ -982,9 +1116,9 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow return; } - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { return; - } + } tscDebug("%p all subquery retrieve ts complete, do ts block intersect", pParentSql); @@ -1022,13 +1156,26 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR SSqlRes* pRes = &pSql->res; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + + tscAsyncResultOnError(pParentSql); + + return; + } + + if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { assert(numOfRows == taos_errno(pSql)); pParentSql->res.code = numOfRows; tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows)); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1061,9 +1208,8 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } } - assert(pState->numOfRemain > 0); - if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { - tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub); + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { + tscDebug("%p sub:%p,%d completed, total:%d", pParentSql, tres, pSupporter->subqueryIndex, pState->numOfSub); return; } @@ -1155,7 +1301,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1178,15 +1324,16 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { } } - // get the number of subquery that need to retrieve the next vnode. + if (orderedPrjQuery) { for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { - pSql->subState.numOfRemain++; + subquerySetState(pSub, &pSql->subState, i, 0); } } } + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; @@ -1233,7 +1380,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1243,7 +1390,19 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { // retrieve data from current vnode. tscDebug("%p retrieve data from %d subqueries", pSql, numOfFetch); SJoinSupporter* pSupporter = NULL; - pSql->subState.numOfRemain = numOfFetch; + + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSql1 = pSql->pSubs[i]; + if (pSql1 == NULL) { + continue; + } + + SSqlRes* pRes1 = &pSql1->res; + + if (pRes1->row >= pRes1->numOfRows) { + subquerySetState(pSql1, &pSql->subState, i, 0); + } + } for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSql1 = pSql->pSubs[i]; @@ -1326,7 +1485,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } // restore the offset value for super table query in case of final result. - tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscRestoreFuncForSTableQuery(pQueryInfo); tscFieldInfoUpdateOffset(pQueryInfo); } @@ -1335,16 +1494,21 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { SJoinSupporter* pSupporter = (SJoinSupporter*)param; SSqlObj* pParentSql = pSupporter->pObj; - + // There is only one subquery and table for each subquery. SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + assert(pQueryInfo->numOfTables == 1 && pSql->cmd.numOfClause == 1); // retrieve actual query results from vnode during the second stage join subquery if (pParentSql->res.code != TSDB_CODE_SUCCESS) { tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code); - quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + + tscAsyncResultOnError(pParentSql); return; } @@ -1356,8 +1520,11 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { tscError("%p abort query, code:%s, global code:%s", pSql, tstrerror(code), tstrerror(pParentSql->res.code)); pParentSql->res.code = code; - quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + + tscAsyncResultOnError(pParentSql); return; } @@ -1378,14 +1545,11 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { return; } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - // In case of consequence query from other vnode, do not wait for other query response here. if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) { - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { return; - } + } } tscSetupOutputColumnIndex(pParentSql); @@ -1397,13 +1561,14 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data pSql->cmd.command = TSDB_SQL_FETCH; + tscProcessSql(pSql); } else { // first retrieve from vnode during the secondary stage sub-query // set the command flag must be after the semaphore been correctly set. if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1432,8 +1597,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pSql->pSubs[pSql->subState.numOfRemain++] = pNew; - assert(pSql->subState.numOfRemain <= pSql->subState.numOfSub); + pSql->pSubs[tableIndex] = pNew; if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { addGroupInfoForSubquery(pSql, pNew, 0, tableIndex); @@ -1502,25 +1666,25 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter // set get tags query type TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG); + tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG); size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscDebug( "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), " "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, tagIndex:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, pNewQueryInfo->pTableMetaInfo[0]->name); + numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); } else { SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; SColumnIndex colIndex = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL); // set the tags value for ts_comp function SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); - pExpr->param->i64Key = tagColId; + pExpr->param->i64 = tagColId; pExpr->numOfParams = 1; } @@ -1544,7 +1708,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%u, transfer to ts_comp query to retrieve timestamps, " "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); + numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); } } else { assert(0); @@ -1565,6 +1729,19 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { int32_t code = TSDB_CODE_SUCCESS; pSql->subState.numOfSub = pQueryInfo->numOfTables; + if (pSql->subState.states == NULL) { + pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); + if (pSql->subState.states == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + pthread_mutex_init(&pSql->subState.mutex, NULL); + } + + memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); + tscDebug("%p reset all sub states to 0", pSql); + bool hasEmptySub = false; tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables); @@ -1597,14 +1774,25 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; (*pSql->fp)(pSql->param, pSql, 0); } else { + int fail = 0; for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; + if (fail) { + (*pSub->fp)(pSub->param, pSub, 0); + continue; + } + if ((code = tscProcessSql(pSub)) != TSDB_CODE_SUCCESS) { - pSql->subState.numOfRemain = i - 1; // the already sent request will continue and do not go to the error process routine - break; + pRes->code = code; + (*pSub->fp)(pSub->param, pSub, 0); + fail = 1; } } + if(fail) { + return; + } + pSql->cmd.command = TSDB_SQL_TABLE_JOIN_RETRIEVE; } @@ -1612,7 +1800,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { _error: pRes->code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { @@ -1631,6 +1819,341 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { } } +void tscLockByThread(int64_t *lockedBy) { + int64_t tid = taosGetSelfPthreadId(); + int i = 0; + while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { + if (++i % 100 == 0) { + sched_yield(); + } + } +} + +void tscUnlockByThread(int64_t *lockedBy) { + int64_t tid = taosGetSelfPthreadId(); + if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { + assert(false); + } +} + +typedef struct SFirstRoundQuerySup { + SSqlObj *pParent; + int32_t numOfRows; + SArray *pColsInfo; + int32_t tagLen; + STColumn *pTagCols; + SArray *pResult; // SArray + int64_t interval; + char* buf; + int32_t bufLen; +} SFirstRoundQuerySup; + +void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, SQueryInfo* pQueryInfo) { + TSKEY key = INT64_MIN; + for(int32_t i = 0; i < numOfCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag) || pExpr->functionId == TSDB_FUNC_PRJ) { + continue; + } + + if (pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + key = *(TSKEY*) row[i]; + continue; + } + + double v = 0; + if (row[i] != NULL) { + v = *(double*) row[i]; + } else { + SET_DOUBLE_NULL(&v); + } + + int32_t id = pExpr->colInfo.colId; + int32_t numOfQueriedCols = (int32_t) taosArrayGetSize(pInterResult->pResult); + + SArray* p = NULL; + for(int32_t j = 0; j < numOfQueriedCols; ++j) { + SStddevInterResult* pColRes = taosArrayGet(pInterResult->pResult, j); + if (pColRes->colId == id) { + p = pColRes->pResult; + break; + } + } + + //append a new column + if (p == NULL) { + SStddevInterResult t = {.colId = id, .pResult = taosArrayInit(10, sizeof(SResPair)),}; + taosArrayPush(pInterResult->pResult, &t); + p = t.pResult; + } + + SResPair pair = {.avg = v, .key = key}; + taosArrayPush(p, &pair); + } +} + +static void destroySup(SFirstRoundQuerySup* pSup) { + taosArrayDestroyEx(pSup->pResult, freeInterResult); + taosArrayDestroy(pSup->pColsInfo); + tfree(pSup); +} + +void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { + SSqlObj* pSql = (SSqlObj*)tres; + SSqlRes* pRes = &pSql->res; + + SFirstRoundQuerySup* pSup = param; + + SSqlObj* pParent = pSup->pParent; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + int32_t code = taos_errno(pSql); + if (code != TSDB_CODE_SUCCESS) { + destroySup(pSup); + taos_free_result(pSql); + pParent->res.code = code; + tscAsyncResultOnError(pParent); + return; + } + + if (numOfRows > 0) { // the number is not correct for group by column in super table query + TAOS_ROW row = NULL; + int32_t numOfCols = taos_field_count(tres); + + if (pSup->tagLen == 0) { // no tags, all rows belong to one group + SInterResult interResult = {.tags = NULL, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; + taosArrayPush(pSup->pResult, &interResult); + + while ((row = taos_fetch_row(tres)) != NULL) { + doAppendData(&interResult, row, numOfCols, pQueryInfo); + pSup->numOfRows += 1; + } + } else { // tagLen > 0 + char* p = calloc(1, pSup->tagLen); + + while ((row = taos_fetch_row(tres)) != NULL) { + int32_t* length = taos_fetch_lengths(tres); + memset(p, 0, pSup->tagLen); + + int32_t offset = 0; + for (int32_t i = 0; i < numOfCols && offset < pSup->tagLen; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + + // tag or group by column + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag) || pExpr->functionId == TSDB_FUNC_PRJ) { + memcpy(p + offset, row[i], length[i]); + offset += pExpr->resBytes; + } + } + + assert(offset == pSup->tagLen); + size_t size = taosArrayGetSize(pSup->pResult); + + if (size > 0) { + SInterResult* pInterResult = taosArrayGetLast(pSup->pResult); + if (memcmp(pInterResult->tags, p, pSup->tagLen) == 0) { // belongs to the same group + doAppendData(pInterResult, row, numOfCols, pQueryInfo); + } else { + char* tags = malloc( pSup->tagLen); + memcpy(tags, p, pSup->tagLen); + + SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; + taosArrayPush(pSup->pResult, &interResult); + doAppendData(&interResult, row, numOfCols, pQueryInfo); + } + } else { + char* tags = malloc(pSup->tagLen); + memcpy(tags, p, pSup->tagLen); + + SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; + taosArrayPush(pSup->pResult, &interResult); + doAppendData(&interResult, row, numOfCols, pQueryInfo); + } + + pSup->numOfRows += 1; + } + + tfree(p); + } + } + + if (!pRes->completed) { + taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); + return; + } + + // set the parameters for the second round query process + SSqlCmd *pPCmd = &pParent->cmd; + SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(pPCmd, 0); + int32_t resRows = pSup->numOfRows; + + if (pSup->numOfRows > 0) { + SBufferWriter bw = tbufInitWriter(NULL, false); + interResToBinary(&bw, pSup->pResult, pSup->tagLen); + + pQueryInfo1->bufLen = (int32_t) tbufTell(&bw); + pQueryInfo1->buf = tbufGetData(&bw, true); + + // set the serialized binary string as the parameter of arithmetic expression + tbufCloseWriter(&bw); + } + + taosArrayDestroyEx(pSup->pResult, freeInterResult); + taosArrayDestroy(pSup->pColsInfo); + tfree(pSup); + + taos_free_result(pSql); + + if (resRows == 0) { + pParent->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + (*pParent->fp)(pParent->param, pParent, 0); + return; + } + + pQueryInfo1->round = 1; + tscDoQuery(pParent); +} + +void tscFirstRoundCallback(void* param, TAOS_RES* tres, int code) { + SFirstRoundQuerySup* pSup = (SFirstRoundQuerySup*) param; + + SSqlObj* pSql = (SSqlObj*) tres; + int32_t c = taos_errno(pSql); + + if (c != TSDB_CODE_SUCCESS) { + SSqlObj* parent = pSup->pParent; + + destroySup(pSup); + taos_free_result(pSql); + parent->res.code = code; + tscAsyncResultOnError(parent); + return; + } + + taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); +} + +int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + STableMetaInfo* pTableMetaInfo1 = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + + SFirstRoundQuerySup *pSup = calloc(1, sizeof(SFirstRoundQuerySup)); + + pSup->pParent = pSql; + pSup->interval = pQueryInfo->interval.interval; + pSup->pResult = taosArrayInit(6, sizeof(SStddevInterResult)); + pSup->pColsInfo = taosArrayInit(6, sizeof(int16_t)); // result column id + + SSqlObj *pNew = createSubqueryObj(pSql, 0, tscFirstRoundCallback, pSup, TSDB_SQL_SELECT, NULL); + SSqlCmd *pCmd = &pNew->cmd; + + tscClearSubqueryInfo(pCmd); + tscFreeSqlResult(pSql); + + SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + assert(pQueryInfo->numOfTables == 1); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); + + tscInitQueryInfo(pNewQueryInfo); + pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; + if (pQueryInfo->groupbyExpr.columnInfo != NULL) { + pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); + if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + } + + if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + pNewQueryInfo->interval = pQueryInfo->interval; + + pCmd->command = TSDB_SQL_SELECT; + pNew->fp = tscFirstRoundCallback; + + int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + + int32_t index = 0; + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_TS && pQueryInfo->interval.interval > 0) { + taosArrayPush(pSup->pColsInfo, &pExpr->resColId); + + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId); + + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL); + p->resColId = pExpr->resColId; // update the result column id + } else if (pExpr->functionId == TSDB_FUNC_STDDEV_DST) { + taosArrayPush(pSup->pColsInfo, &pExpr->resColId); + + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->colInfo.colIndex}; + SSchema schema = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + tstrncpy(schema.name, pExpr->aliasName, tListLen(schema.name)); + + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL); + p->resColId = pExpr->resColId; // update the result column id + } else if (pExpr->functionId == TSDB_FUNC_TAG) { + pSup->tagLen += pExpr->resBytes; + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->colInfo.colIndex}; + + SSchema* schema = NULL; + if (pExpr->colInfo.colId != TSDB_TBNAME_COLUMN_INDEX) { + schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId); + } else { + schema = tGetTbnameColumnSchema(); + } + + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG); + p->resColId = pExpr->resColId; + } else if (pExpr->functionId == TSDB_FUNC_PRJ) { + int32_t num = (int32_t) taosArrayGetSize(pNewQueryInfo->groupbyExpr.columnInfo); + for(int32_t k = 0; k < num; ++k) { + SColIndex* pIndex = taosArrayGet(pNewQueryInfo->groupbyExpr.columnInfo, k); + if (pExpr->colInfo.colId == pIndex->colId) { + pSup->tagLen += pExpr->resBytes; + taosArrayPush(pSup->pColsInfo, &pExpr->resColId); + + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pIndex->colIndex}; + SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId); + + //doLimitOutputNormalColOfGroupby + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL); + p->numOfParams = 1; + p->param[0].i64 = 1; + p->param[0].nType = TSDB_DATA_TYPE_INT; + p->resColId = pExpr->resColId; // update the result column id + } + } + } + } + + SColumnIndex columnIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscInsertPrimaryTsSourceColumn(pNewQueryInfo, &columnIndex); + + tscTansformFuncForSTableQuery(pNewQueryInfo); + + tscDebug( + "%p first round subquery:%p tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, query to retrieve timestamps, " + "numOfExpr:%" PRIzu ", colList:%d, numOfOutputFields:%d, name:%s", + pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pNewQueryInfo->type, + tscSqlExprNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); + + tscHandleMasterSTableQuery(pNew); + return TSDB_CODE_SUCCESS; + + _error: + destroySup(pSup); + taos_free_result(pNew); + pSql->res.code = terrno; + tscAsyncResultOnError(pSql); + return terrno; +} + int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -1641,14 +2164,14 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { return pRes->code; } - tExtMemBuffer ** pMemoryBuf = NULL; + tExtMemBuffer **pMemoryBuf = NULL; tOrderDescriptor *pDesc = NULL; SColumnModel *pModel = NULL; SColumnModel *pFinalModel = NULL; pRes->qhandle = 0x1; // hack the qhandle check - const uint32_t nBufferSize = (1u << 16); // 64KB + const uint32_t nBufferSize = (1u << 16u); // 64KB SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -1666,25 +2189,37 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, &pFinalModel, nBufferSize); if (ret != 0) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); tfree(pMemoryBuf); return ret; } - - pSql->pSubs = calloc(pState->numOfSub, POINTER_BYTES); tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub); - + pSql->pSubs = calloc(pState->numOfSub, POINTER_BYTES); if (pSql->pSubs == NULL) { tfree(pSql->pSubs); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel,pState->numOfSub); - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return ret; } - pState->numOfRemain = pState->numOfSub; + if (pState->states == NULL) { + pState->states = calloc(pState->numOfSub, sizeof(*pState->states)); + if (pState->states == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscAsyncResultOnError(pSql); + tfree(pMemoryBuf); + return ret; + } + + pthread_mutex_init(&pState->mutex, NULL); + } + + memset(pState->states, 0, sizeof(*pState->states) * pState->numOfSub); + tscDebug("%p reset all sub states to 0", pSql); + pRes->code = TSDB_CODE_SUCCESS; int32_t i = 0; @@ -1785,7 +2320,24 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES * current query failed, and the retry count is less than the available * count, retry query clear previous retrieved data, then launch a new sub query */ -static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, int32_t code) { +static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32_t code, int32_t *sent) { + *sent = 0; + + SRetrieveSupport *trsupport = malloc(sizeof(SRetrieveSupport)); + if (trsupport == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + memcpy(trsupport, oriTrs, sizeof(*trsupport)); + + const uint32_t nBufferSize = (1u << 16u); // 64KB + trsupport->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); + if (trsupport->localBuffer == NULL) { + tscError("%p failed to malloc buffer for local buffer, reason:%s", pSql, strerror(errno)); + tfree(trsupport); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SSqlObj *pParentSql = trsupport->pParentSql; int32_t subqueryIndex = trsupport->subqueryIndex; @@ -1802,21 +2354,28 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", - trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); + oriTrs->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, oriTrs->subqueryIndex); pParentSql->res.code = terrno; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + oriTrs->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + tfree(trsupport); return pParentSql->res.code; } int32_t ret = tscProcessSql(pNew); + *sent = 1; + // if failed to process sql, let following code handle the pSql if (ret == TSDB_CODE_SUCCESS) { + tscFreeRetrieveSup(pSql); taos_free_result(pSql); return ret; - } else { + } else { + pParentSql->pSubs[trsupport->subqueryIndex] = pSql; + tscFreeRetrieveSup(pNew); + taos_free_result(pNew); return ret; } } @@ -1833,7 +2392,6 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO assert(pSql != NULL); SSubqueryState* pState = &pParentSql->subState; - assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0); // retrieved in subquery failed. OR query cancelled in retrieve phase. if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) { @@ -1854,7 +2412,10 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { - if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { + int32_t sent = 0; + + tscReissueSubquery(trsupport, pSql, numOfRows, &sent); + if (sent) { return; } } else { // reach the maximum retry count, abort @@ -1864,14 +2425,12 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO } } - int32_t remain = -1; - if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) { - tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex, - pState->numOfSub - remain); + if (!subAndCheckDone(pSql, pParentSql, subqueryIndex)) { + tscDebug("%p sub:%p,%d freed, not finished, total:%d", pParentSql, pSql, trsupport->subqueryIndex, pState->numOfSub); tscFreeRetrieveSup(pSql); return; - } + } // all subqueries are failed tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql, pState->numOfSub, @@ -1890,7 +2449,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); } else { // regular super table query if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1936,14 +2495,12 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p return; } - int32_t remain = -1; - if ((remain = atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1)) > 0) { - tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex, - pState->numOfSub - remain); + if (!subAndCheckDone(pSql, pParentSql, idx)) { + tscDebug("%p sub:%p orderOfSub:%d freed, not finished", pParentSql, pSql, trsupport->subqueryIndex); tscFreeRetrieveSup(pSql); return; - } + } // all sub-queries are returned, start to local merge process pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; @@ -1954,7 +2511,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); tscClearInterpInfo(pPQueryInfo); - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, trsupport->pFFColModel, pParentSql); + tscCreateLocalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, trsupport->pFFColModel, pParentSql); tscDebug("%p build loser tree completed", pParentSql); pParentSql->res.precision = pSql->res.precision; @@ -1968,7 +2525,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } @@ -1980,7 +2537,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SRetrieveSupport *trsupport = (SRetrieveSupport *)param; if (pSql->param == NULL || param == NULL) { tscDebug("%p already freed in dnodecallback", pSql); - assert(pSql->res.code == TSDB_CODE_TSC_QUERY_CANCELLED); return; } @@ -1989,7 +2545,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SSqlObj * pParentSql = trsupport->pParentSql; SSubqueryState* pState = &pParentSql->subState; - assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); SVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; @@ -2013,7 +2568,10 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); - if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { + int32_t sent = 0; + + tscReissueSubquery(trsupport, pSql, numOfRows, &sent); + if (sent) { return; } } else { @@ -2093,7 +2651,15 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo return pNew; } +// todo there is are race condition in this function, while cancel is called by user. void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { + // the param may be null, since it may be done by other query threads. and the asyncOnError may enter in this + // function while kill query by a user. + if (param == NULL) { + assert(code != TSDB_CODE_SUCCESS); + return; + } + SRetrieveSupport *trsupport = (SRetrieveSupport *) param; SSqlObj* pParentSql = trsupport->pParentSql; @@ -2127,7 +2693,11 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); - if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { + + int32_t sent = 0; + + tscReissueSubquery(trsupport, pSql, code, &sent); + if (sent) { return; } } else { @@ -2149,13 +2719,45 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { } } +static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) { + if (pParentObj->retry > pParentObj->maxRetry) { + tscError("%p max retry reached, abort the retry effort", pParentObj); + return false; + } + + for (int32_t i = 0; i < numOfSub; ++i) { + int32_t code = pParentObj->pSubs[i]->res.code; + if (code == TSDB_CODE_SUCCESS) { + continue; + } + + if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE && code != TSDB_CODE_TDB_INVALID_TABLE_ID && + code != TSDB_CODE_VND_INVALID_VGROUP_ID && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && + code != TSDB_CODE_APP_NOT_READY) { + pParentObj->res.code = code; + return false; + } + } + + return true; +} + +static void doFreeInsertSupporter(SSqlObj* pSqlObj) { + assert(pSqlObj != NULL && pSqlObj->subState.numOfSub > 0); + + for(int32_t i = 0; i < pSqlObj->subState.numOfSub; ++i) { + SSqlObj* pSql = pSqlObj->pSubs[i]; + tfree(pSql->param); + } +} + static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) { SInsertSupporter *pSupporter = (SInsertSupporter *)param; SSqlObj* pParentObj = pSupporter->pSql; // record the total inserted rows if (numOfRows > 0) { - pParentObj->res.numOfRows += numOfRows; + atomic_add_fetch_32(&pParentObj->res.numOfRows, numOfRows); } if (taos_errno(tres) != TSDB_CODE_SUCCESS) { @@ -2163,23 +2765,85 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) assert(pSql != NULL && pSql->res.code == numOfRows); pParentObj->res.code = pSql->res.code; - } - tfree(pSupporter); + // set the flag in the parent sqlObj + if (pSql->cmd.submitSchema) { + pParentObj->cmd.submitSchema = 1; + } + } - if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(tres, pParentObj, pSupporter->index)) { + tscDebug("%p insert:%p,%d completed, total:%d", pParentObj, tres, pSupporter->index, pParentObj->subState.numOfSub); return; } - - tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows); // restore user defined fp pParentObj->fp = pParentObj->fetchFp; + int32_t numOfSub = pParentObj->subState.numOfSub; + doFreeInsertSupporter(pParentObj); + + if (pParentObj->res.code == TSDB_CODE_SUCCESS) { + tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows); + + // todo remove this parameter in async callback function definition. + // all data has been sent to vnode, call user function + int32_t v = (pParentObj->res.code != TSDB_CODE_SUCCESS) ? pParentObj->res.code : (int32_t)pParentObj->res.numOfRows; + (*pParentObj->fp)(pParentObj->param, pParentObj, v); + } else { + if (!needRetryInsert(pParentObj, numOfSub)) { + tscAsyncResultOnError(pParentObj); + return; + } + + int32_t numOfFailed = 0; + for(int32_t i = 0; i < numOfSub; ++i) { + SSqlObj* pSql = pParentObj->pSubs[i]; + if (pSql->res.code != TSDB_CODE_SUCCESS) { + numOfFailed += 1; - // todo remove this parameter in async callback function definition. - // all data has been sent to vnode, call user function - int32_t v = (pParentObj->res.code != TSDB_CODE_SUCCESS) ? pParentObj->res.code : (int32_t)pParentObj->res.numOfRows; - (*pParentObj->fp)(pParentObj->param, pParentObj, v); + // clean up tableMeta in cache + tscFreeQueryInfo(&pSql->cmd, false); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); + STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0); + tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); + + subquerySetState(pSql, &pParentObj->subState, i, 0); + + tscDebug("%p, failed sub:%d, %p", pParentObj, i, pSql); + } + } + + tscError("%p Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj, + pParentObj->res.numOfRows, numOfFailed, numOfSub); + + tscDebug("%p cleanup %d tableMeta in hashTable", pParentObj, pParentObj->cmd.numOfTables); + for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(pParentObj->cmd.pTableNameList[i], name); + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + } + + pParentObj->cmd.parseFinished = false; + + tscResetSqlCmd(&pParentObj->cmd, false); + + // in case of insert, redo parsing the sql string and build new submit data block for two reasons: + // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. + // 2. vnode may need the schema information along with submit block to update its local table schema. + tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry); + pParentObj->retry++; + + int32_t code = tsParseSql(pParentObj, true); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; + + if (code != TSDB_CODE_SUCCESS) { + pParentObj->res.code = code; + tscAsyncResultOnError(pParentObj); + return; + } + + tscDoQuery(pParentObj); + } } /** @@ -2187,22 +2851,18 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) * @param pSql * @return */ -int32_t tscHandleInsertRetry(SSqlObj* pSql) { +int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { assert(pSql != NULL && pSql->param != NULL); - SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param; assert(pSupporter->index < pSupporter->pSql->subState.numOfSub); - STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index); + STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.pDataBlocks, pSupporter->index); int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); - // free the data block created from insert sql string - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; // here the pSql may have been released already. } @@ -2213,6 +2873,24 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; + // it is the failure retry insert + if (pSql->pSubs != NULL) { + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + SInsertSupporter* pSup = calloc(1, sizeof(SInsertSupporter)); + pSup->index = i; + pSup->pSql = pSql; + + pSub->param = pSup; + tscDebug("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, i); + if (pSub->res.code != TSDB_CODE_SUCCESS) { + tscHandleInsertRetry(pSql, pSub); + } + } + + return TSDB_CODE_SUCCESS; + } + pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->pDataBlocks); assert(pSql->subState.numOfSub > 0); @@ -2221,7 +2899,19 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { // the number of already initialized subqueries int32_t numOfSub = 0; - pSql->subState.numOfRemain = pSql->subState.numOfSub; + if (pSql->subState.states == NULL) { + pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); + if (pSql->subState.states == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + pthread_mutex_init(&pSql->subState.mutex, NULL); + } + + memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); + tscDebug("%p reset all sub states to 0", pSql); + pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); if (pSql->pSubs == NULL) { goto _error; @@ -2377,7 +3067,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SSqlRes* pRes = &pSql->res; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -2392,19 +3082,19 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } - tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscRestoreFuncForSTableQuery(pQueryInfo); } - while (1) { - assert (pRes->row >= pRes->numOfRows); - - doBuildResFromSubqueries(pSql); - tsem_post(&pSql->rspSem); - return; + assert (pRes->row >= pRes->numOfRows); + doBuildResFromSubqueries(pSql); + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + } else { + tscAsyncResultOnError(pSql); } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 03b6ac8404c1f3d55a1afce81342c497101a6589..bd79f81846cbfd730fd9b3134b89ba0b328472a1 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -17,7 +17,7 @@ #include "taosmsg.h" #include "tref.h" #include "trpc.h" -#include "tsystem.h" +#include "tnote.h" #include "ttimer.h" #include "tutil.h" #include "tsched.h" @@ -30,55 +30,90 @@ #include "tlocale.h" // global, not configurable -SCacheObj* tscMetaCache; -int tscObjRef = -1; -void * tscTmr; -void * tscQhandle; -void * tscCheckDiskUsageTmr; -int tsInsertHeadSize; -int tscRefId = -1; - -int tscNumOfThreads; - +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +int32_t sentinel = TSC_VAR_NOT_RELEASE; + +SHashObj *tscVgroupMap; // hash map to keep the global vgroup info +SHashObj *tscTableMetaInfo; // table meta info +int32_t tscObjRef = -1; +void *tscTmr; +void *tscQhandle; +int32_t tscRefId = -1; +int32_t tscNumOfObj = 0; // number of sqlObj in current process. +static void *tscCheckDiskUsageTmr; +void *tscRpcCache; // cache to keep rpc obj +int32_t tscNumOfThreads = 1; // num of rpc threads +static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); -//void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet); +static volatile int tscInitRes = 0; -void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { +void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) { taosGetDisk(); - taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); + taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); +} +void tscFreeRpcObj(void *param) { + assert(param); + SRpcObj *pRpcObj = (SRpcObj *)(param); + tscDebug("free rpcObj:%p and free pDnodeConn: %p", pRpcObj, pRpcObj->pDnodeConn); + rpcClose(pRpcObj->pDnodeConn); } -int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeConn) { - SRpcInit rpcInit; - - if (*pDnodeConn == NULL) { - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "TSC"; - rpcInit.numOfThreads = 1; // every DB connection has only one thread - rpcInit.cfp = tscProcessMsgFromServer; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char *)user; - rpcInit.idleTime = 2000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char *)secretEncrypt; - - *pDnodeConn = rpcOpen(&rpcInit); - if (*pDnodeConn == NULL) { - tscError("failed to init connection to TDengine"); - return -1; - } else { - tscDebug("dnodeConn:%p is created, user:%s", *pDnodeConn, user); - } +void tscReleaseRpc(void *param) { + if (param == NULL) { + return; } + pthread_mutex_lock(&rpcObjMutex); + taosCacheRelease(tscRpcCache, (void *)¶m, false); + pthread_mutex_unlock(&rpcObjMutex); +} + +int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt, void **ppRpcObj) { + pthread_mutex_lock(&rpcObjMutex); + SRpcObj *pRpcObj = (SRpcObj *)taosCacheAcquireByKey(tscRpcCache, key, strlen(key)); + if (pRpcObj != NULL) { + *ppRpcObj = pRpcObj; + pthread_mutex_unlock(&rpcObjMutex); + return 0; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = tscNumOfThreads; + rpcInit.cfp = tscProcessMsgFromServer; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)user; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.secret = (char *)secretEncrypt; + + SRpcObj rpcObj; + memset(&rpcObj, 0, sizeof(rpcObj)); + strncpy(rpcObj.key, key, strlen(key)); + rpcObj.pDnodeConn = rpcOpen(&rpcInit); + if (rpcObj.pDnodeConn == NULL) { + pthread_mutex_unlock(&rpcObjMutex); + tscError("failed to init connection to TDengine"); + return -1; + } + pRpcObj = taosCachePut(tscRpcCache, rpcObj.key, strlen(rpcObj.key), &rpcObj, sizeof(rpcObj), 1000*5); + if (pRpcObj == NULL) { + rpcClose(rpcObj.pDnodeConn); + pthread_mutex_unlock(&rpcObjMutex); + return -1; + } + + *ppRpcObj = pRpcObj; + pthread_mutex_unlock(&rpcObjMutex); return 0; } - void taos_init_imp(void) { char temp[128] = {0}; @@ -103,7 +138,12 @@ void taos_init_imp(void) { } taosReadGlobalCfg(); - taosCheckGlobalCfg(); + if (taosCheckGlobalCfg()) { + tscInitRes = -1; + return; + } + + taosInitNotes(); rpcInit(); tscDebug("starting to initialize TAOS client ..."); @@ -111,11 +151,6 @@ void taos_init_imp(void) { } taosSetCoreDump(); - - if (tsTscEnableRecordSql != 0) { - taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); - } - tscInitMsgsFp(); int queueSize = tsMaxConnections*2; @@ -124,23 +159,28 @@ void taos_init_imp(void) { if (tscNumOfThreads < 2) { tscNumOfThreads = 2; } - tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc"); if (NULL == tscQhandle) { tscError("failed to init scheduler"); + tscInitRes = -1; return; } tscTmr = taosTmrInit(tsMaxConnections * 2, 200, 60000, "TSC"); if(0 == tscEmbedded){ - taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); + taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); } - int64_t refreshTime = 10; // 10 seconds by default - if (tscMetaCache == NULL) { - tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); - tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + if (tscTableMetaInfo == NULL) { + tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + tscTableMetaInfo = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tscDebug("TableMeta:%p", tscTableMetaInfo); } + + int refreshTime = 5; + tscRpcCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, true, tscFreeRpcObj, "rpcObj"); + pthread_mutex_init(&rpcObjMutex, NULL); tscRefId = taosOpenRef(200, tscCloseTscObj); @@ -151,36 +191,52 @@ void taos_init_imp(void) { tscDebug("client is initialized successfully"); } -void taos_init() { pthread_once(&tscinit, taos_init_imp); } +int taos_init() { pthread_once(&tscinit, taos_init_imp); return tscInitRes;} // this function may be called by user or system, or by both simultaneously. void taos_cleanup(void) { tscDebug("start to cleanup client environment"); - void* m = tscMetaCache; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) { - taosCacheCleanup(m); + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; } - int refId = atomic_exchange_32(&tscObjRef, -1); - if (refId != -1) { - taosCloseRef(refId); - } + taosHashCleanup(tscTableMetaInfo); + tscTableMetaInfo = NULL; - m = tscQhandle; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) { - taosCleanUpScheduler(m); - } + taosHashCleanup(tscVgroupMap); + tscVgroupMap = NULL; + + int32_t id = tscObjRef; + tscObjRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscRefId; + tscRefId = -1; + taosCloseRef(id); - taosCloseRef(tscRefId); taosCleanupKeywordsTable(); - taosCloseLog(); - if (tscEmbedded == 0) rpcCleanup(); - m = tscTmr; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) { - taosTmrCleanUp(m); + p = tscRpcCache; + tscRpcCache = NULL; + + if (p != NULL) { + taosCacheCleanup(p); + pthread_mutex_destroy(&rpcObjMutex); } + + if (tscEmbedded == 0) { + rpcCleanup(); + taosCloseLog(); + }; + + p = tscTmr; + tscTmr = NULL; + taosTmrCleanUp(p); } static int taos_options_imp(TSDB_OPTION option, const char *pStr) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a98132d319cf5403503c415d99c5fd0a72fc1941..578a7df1493b23792791a29bf0acf43c6e38b719 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -16,9 +16,8 @@ #include "tscUtil.h" #include "hash.h" #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "taosmsg.h" -#include "tcache.h" #include "tkey.h" #include "tmd5.h" #include "tscLocalMerge.h" @@ -31,8 +30,16 @@ #include "ttokendef.h" static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); -static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); +static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta); +static void tscStrToLower(char *str, int32_t n) { + if (str == NULL || n <= 0) { return;} + for (int32_t i = 0; i < n; i++) { + if (str[i] >= 'A' && str[i] <= 'Z') { + str[i] -= ('A' - 'a'); + } + } +} SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { return NULL; @@ -72,6 +79,7 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) { bool tscQueryTags(SQueryInfo* pQueryInfo) { int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfCols; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t functId = pExpr->functionId; @@ -89,21 +97,6 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) { return true; } -// todo refactor, extract methods and move the common module -void tscGetDBInfoFromTableFullName(char* tableId, char* db) { - char* st = strstr(tableId, TS_PATH_DELIMITER); - if (st != NULL) { - char* end = strstr(st + 1, TS_PATH_DELIMITER); - if (end != NULL) { - memcpy(db, tableId, (end - tableId)); - db[end - tableId] = 0; - return; - } - } - - db[0] = 0; -} - bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { if (pQueryInfo == NULL) { return false; @@ -114,11 +107,6 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { return false; } - // for select query super table, the super table vgroup list can not be null in any cases. - // if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - // assert(pTableMetaInfo->vgroupList != NULL); - // } - if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { return false; } @@ -315,7 +303,7 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { } else { for (int32_t k = 0; k < pRes->numOfRows; ++k) { char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - memcpy(p, &pInfo->pSqlExpr->param[1].i64Key, pInfo->field.bytes); + memcpy(p, &pInfo->pSqlExpr->param[1].i64, pInfo->field.bytes); } } } @@ -333,13 +321,15 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { if (isNull(p, TSDB_DATA_TYPE_NCHAR)) { memcpy(dst, p, varDataTLen(p)); - } else { + } else if (varDataLen(p) > 0) { int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); varDataSetLen(dst, length); if (length == 0) { tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); } + } else { + varDataSetLen(dst, 0); } p += pInfo->field.bytes; @@ -377,17 +367,16 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } for (int32_t i = 0; i < pCmd->numOfClause; ++i) { - char* addr = (char*)pCmd - offsetof(SSqlObj, cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache); + clearAllTableMetaInfo(pQueryInfo, removeMeta); tfree(pQueryInfo); } @@ -395,7 +384,7 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { tfree(pCmd->pQueryInfo); } -void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { +void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; @@ -403,17 +392,23 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->msgType = 0; pCmd->parseFinished = 0; pCmd->autoCreated = 0; - - taosHashCleanup(pCmd->pTableList); - pCmd->pTableList = NULL; - - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd, removeFromCache); + for(int32_t i = 0; i < pCmd->numOfTables; ++i) { + if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { + tfree(pCmd->pTableNameList[i]); + } + } + + pCmd->numOfTables = 0; + tfree(pCmd->pTableNameList); + + pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, removeMeta); + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + tscFreeQueryInfo(pCmd, removeMeta); } void tscFreeSqlResult(SSqlObj* pSql) { - tscDestroyLocalReducer(pSql); + tscDestroyLocalMerger(pSql); SSqlRes* pRes = &pSql->res; tscDestroyResPointerInfo(pRes); @@ -434,6 +429,12 @@ static void tscFreeSubobj(SSqlObj* pSql) { pSql->pSubs[i] = NULL; } + if (pSql->subState.states) { + pthread_mutex_destroy(&pSql->subState.mutex); + } + + tfree(pSql->subState.states); + pSql->subState.numOfSub = 0; } @@ -449,35 +450,26 @@ void tscFreeRegisteredSqlObj(void *pSql) { SSqlObj* p = *(SSqlObj**)pSql; STscObj* pTscObj = p->pTscObj; + assert(RID_VALID(p->self)); - assert(p->self != 0); - tscFreeSqlObj(p); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfObj, 1); + int32_t total = atomic_sub_fetch_32(&tscNumOfObj, 1); - int32_t ref = T_REF_DEC(pTscObj); - assert(ref >= 0); + tscDebug("%p free SqlObj, total in tscObj:%d, total:%d", pSql, num, total); + tscFreeSqlObj(p); + taosReleaseRef(tscRefId, pTscObj->rid); - tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref); - if (ref == 0) { - tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj); - taosRemoveRef(tscRefId, pTscObj->rid); - } } -void tscFreeTableMetaHelper(void *pTableMeta) { - STableMeta* p = (STableMeta*) pTableMeta; - - int32_t numOfEps = p->vgroupInfo.numOfEps; - assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA); - - for(int32_t i = 0; i < numOfEps; ++i) { - tfree(p->vgroupInfo.epAddr[i].fqdn); - } - - int32_t numOfEps1 = p->corVgroupInfo.numOfEps; - assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA); +void tscFreeMetaSqlObj(int64_t *rid){ + if (RID_VALID(*rid)) { + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, *rid); + if (pSql) { + taosRemoveRef(tscObjRef, *rid); + taosReleaseRef(tscObjRef, *rid); + } - for(int32_t i = 0; i < numOfEps1; ++i) { - tfree(p->corVgroupInfo.epAddr[i].fqdn); + *rid = 0; } } @@ -490,6 +482,9 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + tscFreeMetaSqlObj(&pSql->metaRid); + tscFreeMetaSqlObj(&pSql->svgroupRid); + tscFreeSubobj(pSql); SSqlCmd* pCmd = &pSql->cmd; @@ -508,19 +503,21 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->self = 0; tscFreeSqlResult(pSql); - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmd(pCmd, false); - tfree(pCmd->pTagData); + tfree(pCmd->tagData.data); + pCmd->tagData.dataLen = 0; memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); pCmd->allocSize = 0; tsem_destroy(&pSql->rspSem); + memset(pSql, 0, sizeof(*pSql)); free(pSql); } -void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { +void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { if (pDataBlock == NULL) { return; } @@ -530,7 +527,14 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); + tfree(pDataBlock->pTableMeta); + } + + if (removeMeta) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pDataBlock->tableName, name); + + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } tfree(pDataBlock); @@ -568,13 +572,28 @@ void* tscDestroyBlockArrayList(SArray* pDataBlockList) { size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { void* d = taosArrayGetP(pDataBlockList, i); - tscDestroyDataBlock(d); + tscDestroyDataBlock(d, false); } taosArrayDestroy(pDataBlockList); return NULL; } +void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) { + if (pBlockHashTable == NULL) { + return NULL; + } + + STableDataBlocks** p = taosHashIterate(pBlockHashTable, NULL); + while(p) { + tscDestroyDataBlock(*p, removeMeta); + p = taosHashIterate(pBlockHashTable, p); + } + + taosHashCleanup(pBlockHashTable); + return NULL; +} + int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { SSqlCmd* pCmd = &pSql->cmd; assert(pDataBlock->pTableMeta != NULL); @@ -584,17 +603,16 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { assert(pCmd->numOfClause == 1); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + // todo refactor // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { - tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name)); + tNameAssign(&pTableMetaInfo->name, &pDataBlock->tableName); if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); + tfree(pTableMetaInfo->pTableMeta); } - pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta); - } else { - assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); + pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); } /* @@ -629,7 +647,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { * @param dataBlocks * @return */ -int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, +int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { @@ -657,38 +675,34 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff dataBuf->size = startOffset; dataBuf->tsSource = -1; - tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId)); + tNameAssign(&dataBuf->tableName, name); - /* - * The table meta may be released since the table meta cache are completed clean by other thread - * due to operation such as drop database. So here we add the reference count directly instead of invoke - * taosGetDataFromCache, which may return NULL value. - */ - dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta); + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; return TSDB_CODE_SUCCESS; } -int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks) { +int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, + SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { *dataBlocks = *t1; } if (*dataBlocks == NULL) { - int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, tableId, pTableMeta, dataBlocks); + int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks); if (ret != TSDB_CODE_SUCCESS) { return ret; } taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); - taosArrayPush(pDataBlockList, dataBlocks); + if (pBlockList) { + taosArrayPush(pBlockList, dataBlocks); + } } return TSDB_CODE_SUCCESS; @@ -769,22 +783,44 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { return result; } -int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { +static void extractTableNameList(SSqlCmd* pCmd, bool freeBlockMap) { + pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList); + if (pCmd->pTableNameList == NULL) { + pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES); + } else { + memset(pCmd->pTableNameList, 0, pCmd->numOfTables * POINTER_BYTES); + } + + STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL); + int32_t i = 0; + while(p1) { + STableDataBlocks* pBlocks = *p1; + pCmd->pTableNameList[i++] = tNameDup(&pBlocks->tableName); + p1 = taosHashIterate(pCmd->pTableBlockHashList, p1); + } + + if (freeBlockMap) { + pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, false); + } +} + +int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { + const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); SSqlCmd* pCmd = &pSql->cmd; void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - size_t total = taosArrayGetSize(pTableDataBlockList); - for (int32_t i = 0; i < total; ++i) { + STableDataBlocks** p = taosHashIterate(pCmd->pTableBlockHashList, NULL); + + STableDataBlocks* pOneTableBlock = *p; + while(pOneTableBlock) { // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); STableDataBlocks* dataBuf = NULL; - int32_t ret = - tscGetDataBlockFromList(pVnodeDataBlockHashList, pVnodeDataBlockList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf); + int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, + INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); taosHashCleanup(pVnodeDataBlockHashList); @@ -817,8 +853,8 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { tscSortRemoveDataBlockDupRows(pOneTableBlock); char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - - tscDebug("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, + + tscDebug("%p name:%s, name:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); @@ -839,14 +875,19 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { // the length does not include the SSubmitBlk structure pBlocks->dataLen = htonl(finalLen); dataBuf->numOfTables += 1; + + p = taosHashIterate(pCmd->pTableBlockHashList, p); + if (p == NULL) { + break; + } + + pOneTableBlock = *p; } - tscDestroyBlockArrayList(pTableDataBlockList); + extractTableNameList(pCmd, freeBlockMap); // free the table data blocks; pCmd->pDataBlocks = pVnodeDataBlockList; - -// tscFreeUnusedDataBlocks(pCmd->pDataBlocks); taosHashCleanup(pVnodeDataBlockHashList); return TSDB_CODE_SUCCESS; @@ -859,15 +900,10 @@ void tscCloseTscObj(void *param) { pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - void* p = pObj->pDnodeConn; - if (pObj->pDnodeConn != NULL) { - rpcClose(pObj->pDnodeConn); - pObj->pDnodeConn = NULL; - } tfree(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pObj->pRpcObj); pthread_mutex_destroy(&pObj->mutex); - tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); tfree(pObj); } @@ -1014,7 +1050,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { SInternalField* pInfo = taosArrayGet(pFieldInfo->internalField, i); if (pInfo->pArithExprInfo != NULL) { - tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); + tExprTreeDestroy(pInfo->pArithExprInfo->pExpr, NULL); SSqlFuncMsg* pFuncMsg = &pInfo->pArithExprInfo->base; for(int32_t j = 0; j < pFuncMsg->numOfParams; ++j) { @@ -1033,7 +1069,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { memset(pFieldInfo, 0, sizeof(SFieldInfo)); } -static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +static SSqlExpr* doCreateSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); @@ -1047,6 +1083,8 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else if (pColIndex->columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) { + pExpr->colInfo.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { pExpr->colInfo.colId = pColIndex->columnIndex; } else { @@ -1084,14 +1122,14 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); } - SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); + SSqlExpr* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); taosArrayInsert(pQueryInfo->exprList, index, &pExpr); return pExpr; } SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { - SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); + SSqlExpr* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); taosArrayPush(pQueryInfo->exprList, &pExpr); return pExpr; } @@ -1115,6 +1153,22 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi return pExpr; } +bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) { + return false; + } + + int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_STDDEV_DST) { + return true; + } + } + + return false; +} + size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) { return taosArrayGetSize(pQueryInfo->exprList); } @@ -1260,7 +1314,7 @@ SColumn* tscColumnClone(const SColumn* src) { dst->colIndex = src->colIndex; dst->numOfFilters = src->numOfFilters; - dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); + dst->filterInfo = tFilterInfoDup(src->filterInfo, src->numOfFilters); return dst; } @@ -1378,9 +1432,11 @@ int32_t tscValidateName(SStrToken* pToken) { char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); if (sep == NULL) { // single part if (pToken->type == TK_STRING) { - strdequote(pToken->z); - pToken->n = (uint32_t)strtrim(pToken->z); - + + tscDequoteAndTrimToken(pToken); + tscStrToLower(pToken->z, pToken->n); + //pToken->n = (uint32_t)strtrim(pToken->z); + int len = tSQLGetToken(pToken->z, &pToken->type); // single token, validate it @@ -1432,7 +1488,7 @@ int32_t tscValidateName(SStrToken* pToken) { if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - + // re-build the whole name string if (pStr[firstPartLen] == TS_PATH_DELIMITER[0]) { // first part do not have quote do nothing @@ -1444,6 +1500,8 @@ int32_t tscValidateName(SStrToken* pToken) { } pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0])); pToken->z = pStr; + + tscStrToLower(pToken->z,pToken->n); } return TSDB_CODE_SUCCESS; @@ -1463,7 +1521,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_BLOCK_DIST_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -1492,6 +1550,7 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { } dest->tbnameCond.uid = src->tbnameCond.uid; + dest->tbnameCond.len = src->tbnameCond.len; memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); dest->relType = src->relType; @@ -1714,6 +1773,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); tfree(pQueryInfo->fillVal); + tfree(pQueryInfo->buf); } void tscClearSubqueryInfo(SSqlCmd* pCmd) { @@ -1765,10 +1825,10 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); } - info->itemList = taosArrayClone(pInfo->itemList); + info->itemList = taosArrayDup(pInfo->itemList); } -SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { +SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) { if (pVgroupTables == NULL) { return NULL; } @@ -1787,21 +1847,26 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { return pa; } -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables); - +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (removeMeta) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + } + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); + tscClearTableMetaInfo(pTableMetaInfo); free(pTableMetaInfo); } tfree(pQueryInfo->pTableMetaInfo); } -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { @@ -1819,7 +1884,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; if (name != NULL) { - tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name)); + tNameAssign(&pTableMetaInfo->name, name); } pTableMetaInfo->pTableMeta = pTableMeta; @@ -1838,7 +1903,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } - pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables); + pTableMetaInfo->pVgroupTables = tscVgroupTableInfoDup(pVgroupTables); pQueryInfo->numOfTables += 1; return pTableMetaInfo; @@ -1848,14 +1913,12 @@ STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); } -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) { if (pTableMetaInfo == NULL) { return; } - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - } + tfree(pTableMetaInfo->pTableMeta); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); @@ -1872,13 +1935,15 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { } void registerSqlObj(SSqlObj* pSql) { - int32_t ref = T_REF_INC(pSql->pTscObj); - tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref); - + taosAcquireRef(tscRefId, pSql->pTscObj->rid); pSql->self = taosAddRef(tscObjRef, pSql); + + int32_t num = atomic_add_fetch_32(&pSql->pTscObj->numOfObj, 1); + int32_t total = atomic_add_fetch_32(&tscNumOfObj, 1); + tscDebug("%p new SqlObj from %p, total in tscObj:%d, total:%d", pSql, pSql->pTscObj, num, total); } -SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) { +SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd) { SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, 0); @@ -1893,46 +1958,40 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pCmd->parseFinished = 1; pCmd->autoCreated = pSql->cmd.autoCreated; - if (pSql->cmd.pTagData != NULL) { - int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); - pNew->cmd.pTagData = calloc(1, size); - if (pNew->cmd.pTagData == NULL) { - tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); - free(pNew); - return NULL; - } - memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); + if (code != TSDB_CODE_SUCCESS) { + tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); + free(pNew); + return NULL; } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pNew->rspSem, 0, 0); +#endif // __APPLE__ tscFreeSqlObj(pNew); return NULL; } - pNew->fp = fp; + pNew->fp = fp; pNew->fetchFp = fp; - pNew->param = param; + pNew->param = param; + pNew->sqlstr = NULL; pNew->maxRetry = TSDB_MAX_REPLICA; - pNew->sqlstr = strdup(pSql->sqlstr); - if (pNew->sqlstr == NULL) { - tscError("%p new subquery failed", pSql); - tscFreeSqlObj(pNew); - return NULL; - } - SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); - + tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); registerSqlObj(pNew); + return pNew; } -static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) { +static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t uid) { int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo); if (numOfOutput == 0) { return; @@ -1973,7 +2032,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p tscFieldInfoUpdateOffset(pNewQueryInfo); } -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { +SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); @@ -1985,17 +2044,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); - pNew->pTscObj = pSql->pTscObj; + pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; + pNew->sqlstr = strdup(pSql->sqlstr); - pNew->sqlstr = strdup(pSql->sqlstr); - if (pNew->sqlstr == NULL) { - tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - SSqlCmd* pnCmd = &pNew->cmd; + SSqlCmd* pnCmd = &pNew->cmd; memcpy(pnCmd, pCmd, sizeof(SSqlCmd)); pnCmd->command = cmd; @@ -2006,7 +2059,13 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->numOfClause = 0; pnCmd->clauseIndex = 0; pnCmd->pDataBlocks = NULL; + + pnCmd->numOfTables = 0; pnCmd->parseFinished = 1; + pnCmd->pTableNameList = NULL; + pnCmd->pTableBlockHashList = NULL; + pnCmd->tagData.data = NULL; + pnCmd->tagData.dataLen = 0; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -2023,16 +2082,28 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->limit = pQueryInfo->limit; pNewQueryInfo->slimit = pQueryInfo->slimit; pNewQueryInfo->order = pQueryInfo->order; + pNewQueryInfo->vgroupLimit = pQueryInfo->vgroupLimit; pNewQueryInfo->tsBuf = NULL; pNewQueryInfo->fillType = pQueryInfo->fillType; pNewQueryInfo->fillVal = NULL; pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->pTableMetaInfo = NULL; - + pNewQueryInfo->bufLen = pQueryInfo->bufLen; + + pNewQueryInfo->buf = malloc(pQueryInfo->bufLen); + if (pNewQueryInfo->buf == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + if (pQueryInfo->bufLen > 0) { + memcpy(pNewQueryInfo->buf, pQueryInfo->buf, pQueryInfo->bufLen); + } + pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; @@ -2076,35 +2147,37 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void goto _error; } - doSetSqlExprAndResultFieldInfo(pQueryInfo, pNewQueryInfo, uid); + doSetSqlExprAndResultFieldInfo(pNewQueryInfo, uid); - pNew->fp = fp; + pNew->fp = fp; pNew->fetchFp = fp; - - pNew->param = param; + pNew->param = param; pNew->maxRetry = TSDB_MAX_REPLICA; - char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { - STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup + STableMeta* pTableMeta = tscTableMetaDup(pTableMetaInfo->pTableMeta); assert(pTableMeta != NULL); - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, - pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pTableMeta, pTableMetaInfo->vgroupList, + pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - - STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); + if (pPrevInfo->pTableMeta && pPrevInfo->pTableMeta->tableType < 0) { + terrno = TSDB_CODE_TSC_APP_ERROR; + goto _error; + } + STableMeta* pPrevTableMeta = tscTableMetaDup(pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } + // this case cannot be happened if (pFinalInfo->pTableMeta == NULL) { - tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); + tscError("%p new subquery failed since no tableMeta, name:%s", pSql, tNameGetTableName(&pTableMetaInfo->name)); if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); @@ -2129,7 +2202,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu "," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), - size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey, + size, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pFinalInfo->name), pNewQueryInfo->window.skey, pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); tscPrintSelectClause(pNew, 0); @@ -2166,12 +2239,12 @@ void tscDoQuery(SSqlObj* pSql) { } if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { - tscProcessMultiVnodesImportFromFile(pSql); + tscImportDataFromFile(pSql); } else { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); uint16_t type = pQueryInfo->type; - if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion + if (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_INSERT)) { // multi-vnodes insertion tscHandleMultivnodeInsert(pSql); return; } @@ -2184,16 +2257,23 @@ void tscDoQuery(SSqlObj* pSql) { tscProcessSql(pSql); } else { // secondary stage join query. if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query + tscLockByThread(&pSql->squeryLock); tscHandleMasterSTableQuery(pSql); + tscUnlockByThread(&pSql->squeryLock); } else { tscProcessSql(pSql); } } } + return; + } else if (tscMultiRoundQuery(pQueryInfo, 0) && pQueryInfo->round == 0) { + tscHandleFirstRoundStableQuery(pSql); // todo lock? return; } else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query + tscLockByThread(&pSql->squeryLock); tscHandleMasterSTableQuery(pSql); + tscUnlockByThread(&pSql->squeryLock); return; } @@ -2237,6 +2317,35 @@ bool tscIsUpdateQuery(SSqlObj* pSql) { return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_USE_DB == pCmd->command); } +char* tscGetSqlStr(SSqlObj* pSql) { + if (pSql == NULL || pSql->signature != pSql) { + return NULL; + } + + return pSql->sqlstr; +} + +bool tscIsQueryWithLimit(SSqlObj* pSql) { + if (pSql == NULL || pSql->signature != pSql) { + return false; + } + + SSqlCmd* pCmd = &pSql->cmd; + for (int32_t i = 0; i < pCmd->numOfClause; ++i) { + SQueryInfo* pqi = tscGetQueryInfoDetailSafely(pCmd, i); + if (pqi == NULL) { + continue; + } + + if (pqi->limit.limit > 0) { + return true; + } + } + + return false; +} + + int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql) { const char* msgFormat1 = "syntax error near \'%s\'"; const char* msgFormat2 = "syntax error near \'%s\' (%s)"; @@ -2262,7 +2371,6 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s } return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { @@ -2477,7 +2585,7 @@ bool tscSetSqlOwner(SSqlObj* pSql) { SSqlRes* pRes = &pSql->res; // set the sql object owner - uint64_t threadId = taosGetPthreadId(); + int64_t threadId = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; @@ -2487,7 +2595,6 @@ bool tscSetSqlOwner(SSqlObj* pSql) { } void tscClearSqlOwner(SSqlObj* pSql) { - assert(taosCheckPthreadValid(pSql->owner)); atomic_store_64(&pSql->owner, 0); } @@ -2531,6 +2638,10 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { tfree(pVgroupInfo->epAddr[j].fqdn); } + + for(int32_t j = pVgroupInfo->numOfEps; j < TSDB_MAX_REPLICA; j++) { + assert( pVgroupInfo->epAddr[j].fqdn == NULL ); + } } tfree(vgroupList); @@ -2543,6 +2654,129 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) { for(int32_t i = 0; i < dst->numOfEps; ++i) { tfree(dst->epAddr[i].fqdn); dst->epAddr[i].port = src->epAddr[i].port; + assert(dst->epAddr[i].fqdn == NULL); + dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); } } + +char* serializeTagData(STagData* pTagData, char* pMsg) { + int32_t n = (int32_t) strlen(pTagData->name); + *(int32_t*) pMsg = htonl(n); + pMsg += sizeof(n); + + memcpy(pMsg, pTagData->name, n); + pMsg += n; + + *(int32_t*)pMsg = htonl(pTagData->dataLen); + pMsg += sizeof(int32_t); + + memcpy(pMsg, pTagData->data, pTagData->dataLen); + pMsg += pTagData->dataLen; + + return pMsg; +} + +int32_t copyTagData(STagData* dst, const STagData* src) { + dst->dataLen = src->dataLen; + tstrncpy(dst->name, src->name, tListLen(dst->name)); + + if (dst->dataLen > 0) { + dst->data = malloc(dst->dataLen); + if (dst->data == NULL) { + return -1; + } + + memcpy(dst->data, src->data, dst->dataLen); + } + + return 0; +} + +STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { + assert(pChild != NULL); + int32_t total = pChild->numOfColumns + pChild->numOfTags; + + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); + pTableMeta->tableType = TSDB_SUPER_TABLE; + pTableMeta->tableInfo.numOfTags = pChild->numOfTags; + pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; + pTableMeta->tableInfo.precision = pChild->precision; + + pTableMeta->id.tid = 0; + pTableMeta->id.uid = pChild->suid; + pTableMeta->tversion = pChild->tversion; + pTableMeta->sversion = pChild->sversion; + + memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); + + int32_t num = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < num; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + int32_t totalCols = 0; + if (pTableMeta->tableInfo.numOfColumns >= 0 && pTableMeta->tableInfo.numOfTags >= 0) { + totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + } + + return sizeof(STableMeta) + totalCols * sizeof(SSchema); +} + +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + return cMeta; +} + +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name) { + assert(pChild != NULL); + + uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* p = calloc(1, size); + + taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); + if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + pChild->sversion = p->sversion; + pChild->tversion = p->tversion; + + memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableInfo)); + int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; + + memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); + + tfree(p); + return TSDB_CODE_SUCCESS; + } else { // super table has been removed, current tableMeta is also expired. remove it here + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + + tfree(p); + return -1; + } +} + +uint32_t tscGetTableMetaMaxSize() { + return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); +} + +STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + uint32_t size = tscGetTableMetaSize(pTableMeta); + STableMeta* p = calloc(1, size); + memcpy(p, pTableMeta, size); + return p; +} + + diff --git a/src/client/tests/resultFieldTest.cpp b/src/client/tests/cliTest.cpp similarity index 58% rename from src/client/tests/resultFieldTest.cpp rename to src/client/tests/cliTest.cpp index c917f0ebafa02480c372e8d7bb611588bad8e54a..30f248b5418b54b1be26dfcf15348b03fd70af4d 100644 --- a/src/client/tests/resultFieldTest.cpp +++ b/src/client/tests/cliTest.cpp @@ -2,15 +2,117 @@ #include #include "taos.h" +#include "tglobal.h" namespace { static int64_t start_ts = 1433955661000; + +void stmtInsertTest() { + TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); + if (conn == NULL) { + printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); + exit(-1); + } + + TAOS_RES* res = taos_query(conn, "use test"); + taos_free_result(res); + + const char* sql = "insert into t1 values(?, ?, ?, ?)"; + TAOS_STMT* stmt = taos_stmt_init(conn); + + int32_t ret = taos_stmt_prepare(stmt, sql, 0); + ASSERT_EQ(ret, 0); + + //ts timestamp, k int, a binary(11), b nchar(4) + struct { + int64_t ts; + int k; + char* a; + char* b; + } v = {0}; + + TAOS_BIND params[4]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(v.k); + params[1].buffer = &v.k; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_BINARY; + params[2].buffer_length = sizeof(v.a); + params[2].buffer = &v.a; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[3].buffer_length = sizeof(v.b); + params[3].buffer = &v.b; + params[3].is_null = NULL; + + v.ts = start_ts + 20; + v.k = 123; + + char str[] = "abc"; + uintptr_t len = strlen(str); + + v.a = str; + params[2].length = &len; + params[2].buffer_length = len; + params[2].buffer = str; + + char nstr[] = "999"; + uintptr_t len1 = strlen(nstr); + + v.b = nstr; + params[3].buffer_length = len1; + params[3].buffer = nstr; + params[3].length = &len1; + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; + } + + v.ts = start_ts + 30; + v.k = 911; + + char str1[] = "92"; + len = strlen(str1); + + params[2].length = &len; + params[2].buffer_length = len; + params[2].buffer = str1; + + char nstr1[] = "1920"; + len1 = strlen(nstr1); + + params[3].buffer_length = len1; + params[3].buffer = nstr1; + params[3].length = &len1; + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + + ret = taos_stmt_execute(stmt); + if (ret != 0) { + printf("%d\n", ret); + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; + } + + taos_stmt_close(stmt); + taos_close(conn); } -/* test parse time function */ -TEST(testCase, result_field_test) { - taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); - taos_init(); +void validateResultFields() { TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); if (conn == NULL) { printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); @@ -134,5 +236,31 @@ TEST(testCase, result_field_test) { ASSERT_STREQ(fields[6].name, "first(ts)"); taos_free_result(res); + + // update the configure parameter, the result field name will be changed + tsKeepOriginalColumnName = 1; + res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); + ASSERT_EQ(taos_num_fields(res), 7); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "ts"); + + ASSERT_EQ(fields[2].bytes, 4); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[2].name, "k"); + + taos_free_result(res); + taos_close(conn); } +} +/* test parse time function */ +TEST(testCase, result_field_test) { + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + taos_init(); + + validateResultFields(); + stmtInsertTest(); +} diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp index bee01b5cecdf5f42a6cd0105fa6b808a23957012..d7325430cd1d859ab9dd3e86f8dd55ab0dd1355d 100644 --- a/src/client/tests/timeParseTest.cpp +++ b/src/client/tests/timeParseTest.cpp @@ -162,6 +162,10 @@ TEST(testCase, parse_time) { taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); + + char* t = "2021-01-08T02:11:40.000+00:00"; + taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0); + printf("%ld\n", time); } diff --git a/src/query/inc/qSyntaxtreefunction.h b/src/common/inc/tarithoperator.h similarity index 67% rename from src/query/inc/qSyntaxtreefunction.h rename to src/common/inc/tarithoperator.h index 6f91d2f7eded38425cf8414b6998e0162557b2c8..27e8871e2f7b32bf7772dfa0cbc9c3164a312f2d 100644 --- a/src/query/inc/qSyntaxtreefunction.h +++ b/src/common/inc/tarithoperator.h @@ -13,17 +13,17 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSYNTAXTREEFUNCTION_H -#define TDENGINE_TSYNTAXTREEFUNCTION_H +#ifndef TDENGINE_QARITHMETICOPERATOR_H +#define TDENGINE_QARITHMETICOPERATOR_H #ifdef __cplusplus extern "C" { #endif -typedef void (*_bi_consumer_fn_t)(void *left, void *right, int32_t numOfLeft, int32_t numOfRight, void *output, - int32_t order); +typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, + int32_t rightType, void *output, int32_t order); -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr); +_arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr); #ifdef __cplusplus } diff --git a/src/common/inc/tcmdtype.h b/src/common/inc/tcmdtype.h index 69bbccd67e79eb77e3afd0523dec765e53a5cfa1..bec85905369b5a1ea4274a9d799c664ce7aee544 100644 --- a/src/common/inc/tcmdtype.h +++ b/src/common/inc/tcmdtype.h @@ -36,7 +36,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" ) - + // the SQL below is for mgmt node TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" ) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8d4949d9b4364fd1e8c70cbb883aa56468724108..959654d1585b13c31e96ab7972d0aa2782a51901 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -20,30 +20,30 @@ #include #include "talgo.h" -#include "taosdef.h" +#include "ttype.h" #include "tutil.h" #ifdef __cplusplus extern "C" { #endif -#define STR_TO_VARSTR(x, str) \ - do { \ - VarDataLenT __len = strlen(str); \ - *(VarDataLenT *)(x) = __len; \ - memcpy(varDataVal(x), (str), __len); \ +#define STR_TO_VARSTR(x, str) \ + do { \ + VarDataLenT __len = (VarDataLenT)strlen(str); \ + *(VarDataLenT *)(x) = __len; \ + memcpy(varDataVal(x), (str), __len); \ } while (0); -#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ - do { \ +#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ + do { \ char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \ - varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ + varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ } while (0) -#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ - do { \ - *(VarDataLenT *)(x) = (_size); \ - memcpy(varDataVal(x), (str), (_size)); \ +#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ + do { \ + *(VarDataLenT *)(x) = (VarDataLenT)(_size); \ + memcpy(varDataVal(x), (str), (_size)); \ } while (0); // ----------------- TSDB COLUMN DEFINITION @@ -68,9 +68,9 @@ typedef struct { typedef struct { int version; // version int numOfCols; // Number of columns appended - int tlen; // maximum length of a SDataRow without the header part + int tlen; // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes)) uint16_t flen; // First part length in a SDataRow after the header part - uint16_t vlen; // pure value part length, excluded the overhead + uint16_t vlen; // pure value part length, excluded the overhead (bytes only) STColumn columns[]; } STSchema; @@ -134,6 +134,22 @@ typedef uint64_t TKEY; #define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) #define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) +#define MIN_TS_KEY ((TSKEY)0x8000000000000001) +#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff) + +#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key)) + +static FORCE_INLINE TKEY keyToTkey(TSKEY key) { + TSKEY lkey = key; + if (key > MAX_TS_KEY) { + lkey = MAX_TS_KEY; + } else if (key < MIN_TS_KEY) { + lkey = MIN_TS_KEY; + } + + return tdGetTKEY(lkey); +} + static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { TSKEY key1 = tdGetKey(*(TKEY *)tkey1); TSKEY key2 = tdGetKey(*(TKEY *)tkey2); @@ -156,7 +172,7 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { * +----------+----------+---------------------------------+---------------------------------+ * | len | sversion | First part | Second part | * +----------+----------+---------------------------------+---------------------------------+ - * + * * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; @@ -278,7 +294,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); -void tdFreeDataCols(SDataCols *pCols); +SDataCols *tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); diff --git a/src/query/inc/qAst.h b/src/common/inc/texpr.h similarity index 77% rename from src/query/inc/qAst.h rename to src/common/inc/texpr.h index 28c1c7b838236d10a3f67fa7bbfc15a9b36c4612..acfbffc01e400f8b111ee92b7651bb048c112bd2 100644 --- a/src/query/inc/qAst.h +++ b/src/common/inc/texpr.h @@ -31,6 +31,15 @@ extern "C" { struct tExprNode; struct SSchema; +#define QUERY_COND_REL_PREFIX_IN "IN|" +#define QUERY_COND_REL_PREFIX_LIKE "LIKE|" + +#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_LIKE_LEN 5 + +typedef bool (*__result_filter_fn_t)(const void *, void *); +typedef void (*__do_filter_suppl_fn_t)(void *, void *); + enum { TSQL_NODE_DUMMY = 0x0, TSQL_NODE_EXPR = 0x1, @@ -38,9 +47,6 @@ enum { TSQL_NODE_VALUE = 0x4, }; -typedef bool (*__result_filter_fn_t)(const void *, void *); -typedef void (*__do_filter_suppl_fn_t)(void *, void *); - /** * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required */ @@ -52,12 +58,6 @@ typedef struct tQueryInfo { bool indexed; // indexed columns } tQueryInfo; -typedef struct SExprTraverseSupp { - __result_filter_fn_t nodeFilterFn; - __do_filter_suppl_fn_t setupInfoFn; - void * pExtInfo; -} SExprTraverseSupp; - typedef struct tExprNode { uint8_t nodeType; union { @@ -65,7 +65,7 @@ typedef struct tExprNode { uint8_t optr; // filter operator uint8_t hasPK; // 0: do not contain primary filter, 1: contain void * info; // support filter operation on this expression only available for leaf node - + struct tExprNode *pLeft; // left child pointer struct tExprNode *pRight; // right child pointer } _node; @@ -74,18 +74,26 @@ typedef struct tExprNode { }; } tExprNode; -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); +typedef struct SExprTraverseSupp { + __result_filter_fn_t nodeFilterFn; + __do_filter_suppl_fn_t setupInfoFn; + void * pExtInfo; +} SExprTraverseSupp; -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*cb)(void *, const char*, int32_t)); +void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); tExprNode* exprTreeFromBinary(const void* data, size_t size); tExprNode* exprTreeFromTableName(const char* tbnameCond); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); -void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); -void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); + +typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, + int32_t rightType, void *output, int32_t order); + +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, + char *(*cb)(void *, const char*, int32_t)); #ifdef __cplusplus } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 851ca57ba9be376ab8cef06567ec8c6f1206a8e4..c6d0226244b9b64d21fcc6c7939d61fa27a55525 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_COMMON_GLOBAL_H #define TDENGINE_COMMON_GLOBAL_H +#include "taosdef.h" + #ifdef __cplusplus extern "C" { #endif @@ -32,9 +34,11 @@ extern uint16_t tsSyncPort; extern uint16_t tsArbitratorPort; extern int32_t tsStatusInterval; extern int32_t tsNumOfMnodes; -extern int32_t tsEnableVnodeBak; -extern int32_t tsEnableTelemetryReporting; +extern int8_t tsEnableVnodeBak; +extern int8_t tsEnableTelemetryReporting; extern char tsEmail[]; +extern char tsArbitrator[]; +extern int8_t tsArbOnline; // common extern int tsRpcTimer; @@ -45,23 +49,25 @@ extern int32_t tsShellActivityTimer; extern uint32_t tsMaxTmrCtrl; extern float tsNumOfThreadsPerCore; extern int32_t tsNumOfCommitThreads; -extern float tsRatioOfQueryThreads; // todo remove it +extern float tsRatioOfQueryCores; extern int8_t tsDaylight; extern char tsTimezone[]; extern char tsLocale[]; extern char tsCharset[]; // default encode string -extern int32_t tsEnableCoreFile; +extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern char tsTempDir[]; //query buffer management -extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing -extern int32_t tsHalfCoresForQuery; // only 50% will be used in query processing +extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing +extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing +extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked + +extern int8_t tsKeepOriginalColumnName; // client -extern int32_t tsTableMetaKeepTimer; extern int32_t tsMaxSQLStringLen; -extern int32_t tsTscEnableRecordSql; +extern int8_t tsTscEnableRecordSql; extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; @@ -85,53 +91,56 @@ extern int32_t tsMinRowsInFileBlock; extern int32_t tsMaxRowsInFileBlock; extern int16_t tsCommitTime; // seconds extern int32_t tsTimePrecision; -extern int16_t tsCompression; -extern int16_t tsWAL; +extern int8_t tsCompression; +extern int8_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; -extern int32_t tsUpdate; +extern int8_t tsUpdate; +extern int8_t tsCacheLastRow; // balance -extern int32_t tsEnableBalance; -extern int32_t tsAlternativeRole; +extern int8_t tsEnableBalance; +extern int8_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; extern int32_t tsMnodeEqualVnodeNum; -extern int32_t tsFlowCtrl; +extern int8_t tsEnableFlowCtrl; +extern int8_t tsEnableSlaveQuery; +extern int8_t tsEnableAdjustMaster; // restful -extern int32_t tsEnableHttpModule; +extern int8_t tsEnableHttpModule; extern int32_t tsRestRowLimit; extern uint16_t tsHttpPort; extern int32_t tsHttpCacheSessions; extern int32_t tsHttpSessionExpire; extern int32_t tsHttpMaxThreads; -extern int32_t tsHttpEnableCompress; -extern int32_t tsHttpEnableRecordSql; -extern int32_t tsTelegrafUseFieldNum; +extern int8_t tsHttpEnableCompress; +extern int8_t tsHttpEnableRecordSql; +extern int8_t tsTelegrafUseFieldNum; // mqtt -extern int32_t tsEnableMqttModule; -extern char tsMqttHostName[]; -extern char tsMqttPort[]; -extern char tsMqttUser[]; -extern char tsMqttPass[]; -extern char tsMqttClientId[]; -extern char tsMqttTopic[]; +extern int8_t tsEnableMqttModule; +extern char tsMqttHostName[]; +extern char tsMqttPort[]; +extern char tsMqttUser[]; +extern char tsMqttPass[]; +extern char tsMqttClientId[]; +extern char tsMqttTopic[]; // monitor -extern int32_t tsEnableMonitorModule; +extern int8_t tsEnableMonitorModule; extern char tsMonitorDbName[]; extern char tsInternalPass[]; extern int32_t tsMonitorInterval; // stream -extern int32_t tsEnableStream; +extern int8_t tsEnableStream; // internal -extern int32_t tsPrintAuth; -extern int32_t tscEmbedded; +extern int8_t tsPrintAuth; +extern int8_t tscEmbedded; extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; @@ -140,7 +149,6 @@ extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; extern int64_t tsMsPerDay[3]; -extern char tsVnodeBakDir[]; // system info extern char tsOsName[]; @@ -158,7 +166,7 @@ extern float tsMinimalLogDirGB; extern float tsReservedTmpDirectorySpace; extern float tsMinimalDataDirGB; extern int32_t tsTotalMemoryMB; -extern int32_t tsVersion; +extern uint32_t tsVersion; // build info extern char version[]; @@ -168,13 +176,13 @@ extern char gitinfoOfInternal[]; extern char buildinfo[]; // log -extern int32_t tsAsyncLog; +extern int8_t tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; extern int32_t vDebugFlag; extern int32_t mDebugFlag; -extern int32_t cDebugFlag; +extern uint32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; extern int32_t sdbDebugFlag; @@ -184,11 +192,19 @@ extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t odbcDebugFlag; -extern int32_t qDebugFlag; +extern uint32_t qDebugFlag; extern int32_t wDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; +typedef struct { + char dir[TSDB_FILENAME_LEN]; + int level; + int primary; +} SDiskCfg; +extern int32_t tsDiskCfgNum; +extern SDiskCfg tsDiskCfg[]; + #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); @@ -197,6 +213,9 @@ void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); +void taosAddDataDir(int index, char *v1, int level, int primary); +void taosReadDataDirCfg(char *v1, char *v2, char *v3); +void taosPrintDataDirCfg(); #ifdef __cplusplus } diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 6c48ca72f3b5ea010c662faa09211afd40152615..b651913d7384bac75eea7c9644b501b781c47e33 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef TDENGINE_NAME_H #define TDENGINE_NAME_H @@ -21,20 +36,79 @@ typedef struct SColumnInfoData { void* pData; // the corresponding block data in memory } SColumnInfoData; +typedef struct SResPair { + TSKEY key; + double avg; +} SResPair; + +#define TSDB_DB_NAME_T 1 +#define TSDB_TABLE_NAME_T 2 + +#define T_NAME_ACCT 0x1u +#define T_NAME_DB 0x2u +#define T_NAME_TABLE 0x4u + +typedef struct SName { + uint8_t type; //db_name_t, table_name_t + char acctId[TSDB_ACCT_ID_LEN]; + char dbname[TSDB_DB_NAME_LEN]; + char tname[TSDB_TABLE_NAME_LEN]; +} SName; + void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); +size_t tableIdPrefix(const char* name, char* prefix, int32_t len); + void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); -SSchema tGetTableNameColumnSchema(); +//SSchema tGetTbnameColumnSchema(); + +SSchema tGetBlockDistColumnSchema(); SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); bool tscValidateTableNameLength(size_t len); -SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); +SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters); + +SSchema* tGetTbnameColumnSchema(); + +/** + * check if the schema is valid or not, including following aspects: + * 1. number of columns + * 2. column types + * 3. column length + * 4. column names + * 5. total length + * + * @param pSchema + * @param numOfCols + * @return + */ +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); + +int32_t tNameExtractFullName(const SName* name, char* dst); +int32_t tNameLen(const SName* name); + +SName* tNameDup(const SName* name); + +bool tIsValidName(const SName* name); + +const char* tNameGetTableName(const SName* name); + +int32_t tNameGetDbName(const SName* name, char* dst); +int32_t tNameGetFullDbName(const SName* name, char* dst); + +bool tNameIsEmpty(const SName* name); + +void tNameAssign(SName* dst, const SName* src); + +int32_t tNameFromString(SName* dst, const char* str, uint32_t type); + +int32_t tNameSetAcctId(SName* dst, const char* acct); -SSchema tscGetTbnameColumnSchema(); +int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken); #endif // TDENGINE_NAME_H diff --git a/src/common/inc/tulog.h b/src/common/inc/tulog.h index 2dc2895e6324053992b5cbdcad5486071b68dbc9..566da40a10e078b9789e2e1b76a8d82fe89aef46 100644 --- a/src/common/inc/tulog.h +++ b/src/common/inc/tulog.h @@ -23,7 +23,7 @@ extern "C" { #include "tlog.h" extern int32_t uDebugFlag; -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index e9973bcb9595c5ace0b4cb20f2d8776a5cf05501..f8f715c6cae1990b3c208c50bd75a1355795671e 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -28,7 +28,8 @@ typedef struct tVariant { uint32_t nType; int32_t nLen; // only used for string, for number, it is useless union { - int64_t i64Key; + int64_t i64; + uint64_t u64; double dKey; char * pz; wchar_t *wpz; @@ -36,9 +37,9 @@ typedef struct tVariant { }; } tVariant; -void tVariantCreate(tVariant *pVar, SStrToken *token); +bool tVariantIsValid(tVariant *pVar); -void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); +void tVariantCreate(tVariant *pVar, SStrToken *token); void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type); diff --git a/src/common/src/tarithoperator.c b/src/common/src/tarithoperator.c new file mode 100644 index 0000000000000000000000000000000000000000..1cb667d259f040cfab0656562f7c97444fc48d8a --- /dev/null +++ b/src/common/src/tarithoperator.c @@ -0,0 +1,2574 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "ttype.h" +#include "tutil.h" +#include "tarithoperator.h" + +#define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ + { \ + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; \ + \ + if ((len1) == (len2)) { \ + for (; i < (len2) && i >= 0; i += step, (out) += 1) { \ + if (isNull((char *)&((left)[i]), _left_type) || isNull((char *)&((right)[i]), _right_type)) { \ + SET_DOUBLE_NULL(out); \ + continue; \ + } \ + *(out) = (double)(left)[i] op(right)[i]; \ + } \ + } else if ((len1) == 1) { \ + for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ + if (isNull((char *)(left), _left_type) || isNull((char *)&(right)[i], _right_type)) { \ + SET_DOUBLE_NULL(out); \ + continue; \ + } \ + *(out) = (double)(left)[0] op(right)[i]; \ + } \ + } else if ((len2) == 1) { \ + for (; i >= 0 && i < (len1); i += step, (out) += 1) { \ + if (isNull((char *)&(left)[i], _left_type) || isNull((char *)(right), _right_type)) { \ + SET_DOUBLE_NULL(out); \ + continue; \ + } \ + *(out) = (double)(left)[i] op(right)[0]; \ + } \ + } \ + } + +#define ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ + { \ + int32_t i = (_ord == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ + int32_t step = (_ord == TSDB_ORDER_ASC) ? 1 : -1; \ + \ + if (len1 == (len2)) { \ + for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ + if (isNull((char *)&(left[i]), _left_type) || isNull((char *)&(right[i]), _right_type)) { \ + SET_DOUBLE_NULL(out); \ + continue; \ + } \ + *(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[i])) * (right)[i]; \ + } \ + } else if (len1 == 1) { \ + for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ + if (isNull((char *)(left), _left_type) || isNull((char *)&((right)[i]), _right_type)) { \ + SET_DOUBLE_NULL(out); \ + continue; \ + } \ + *(out) = (double)(left)[0] - ((int64_t)(((double)(left)[0]) / (right)[i])) * (right)[i]; \ + } \ + } else if ((len2) == 1) { \ + for (; i >= 0 && i < len1; i += step, (out) += 1) { \ + if (isNull((char *)&((left)[i]), _left_type) || isNull((char *)(right), _right_type)) { \ + SET_DOUBLE_NULL(out); \ + continue; \ + } \ + *(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[0])) * (right)[0]; \ + } \ + } \ + } + +#define ARRAY_LIST_ADD(left, right, _left_type, _right_type, len1, len2, out, _ord) \ + ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, +, TSDB_DATA_TYPE_DOUBLE, _ord) +#define ARRAY_LIST_SUB(left, right, _left_type, _right_type, len1, len2, out, _ord) \ + ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, -, TSDB_DATA_TYPE_DOUBLE, _ord) +#define ARRAY_LIST_MULTI(left, right, _left_type, _right_type, len1, len2, out, _ord) \ + ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, *, TSDB_DATA_TYPE_DOUBLE, _ord) +#define ARRAY_LIST_DIV(left, right, _left_type, _right_type, len1, len2, out, _ord) \ + ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, /, TSDB_DATA_TYPE_DOUBLE, _ord) +#define ARRAY_LIST_REM(left, right, _left_type, _right_type, len1, len2, out, _ord) \ + ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, %, TSDB_DATA_TYPE_DOUBLE, _ord) + +#define TYPE_CONVERT_DOUBLE_RES(left, right, out, _type_left, _type_right, _type_res) \ + _type_left * pLeft = (_type_left *)(left); \ + _type_right *pRight = (_type_right *)(right); \ + _type_res * pOutput = (_type_res *)(out); + +#define DO_VECTOR_ADD(left, numLeft, leftType, leftOriginType, right, numRight, rightType, rightOriginType, _output, \ + _order) \ + do { \ + TYPE_CONVERT_DOUBLE_RES(left, right, _output, leftOriginType, rightOriginType, double); \ + ARRAY_LIST_ADD(pLeft, pRight, leftType, rightType, numLeft, numRight, pOutput, _order); \ + } while (0) + +#define DO_VECTOR_SUB(left, numLeft, leftType, leftOriginType, right, numRight, rightType, rightOriginType, _output, \ + _order) \ + do { \ + TYPE_CONVERT_DOUBLE_RES(left, right, _output, leftOriginType, rightOriginType, double); \ + ARRAY_LIST_SUB(pLeft, pRight, leftType, rightType, numLeft, numRight, pOutput, _order); \ + } while (0) + +#define DO_VECTOR_MULTIPLY(left, numLeft, leftType, leftOriginType, right, numRight, rightType, rightOriginType, \ + _output, _order) \ + do { \ + TYPE_CONVERT_DOUBLE_RES(left, right, _output, leftOriginType, rightOriginType, double); \ + ARRAY_LIST_MULTI(pLeft, pRight, leftType, rightType, numLeft, numRight, pOutput, _order); \ + } while (0) + +#define DO_VECTOR_DIVIDE(left, numLeft, leftType, leftOriginType, right, numRight, rightType, rightOriginType, \ + _output, _order) \ + do { \ + TYPE_CONVERT_DOUBLE_RES(left, right, _output, leftOriginType, rightOriginType, double); \ + ARRAY_LIST_DIV(pLeft, pRight, leftType, rightType, numLeft, numRight, pOutput, _order); \ + } while (0) + +#define DO_VECTOR_REMAINDER(left, numLeft, leftType, leftOriginType, right, numRight, rightType, rightOriginType, \ + _output, _order) \ + do { \ + TYPE_CONVERT_DOUBLE_RES(left, right, _output, leftOriginType, rightOriginType, double); \ + ARRAY_LIST_REM(pLeft, pRight, leftType, rightType, numLeft, numRight, pOutput, _order); \ + } while (0) + +void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { + int32_t *pLeft = (int32_t *)left; + int32_t *pRight = (int32_t *)right; + double * pOutput = (double *)output; + + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; + + if (numLeft == numRight) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[i] + pRight[i]; + } + } else if (numLeft == 1) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[0] + pRight[i]; + } + } else if (numRight == 1) { + for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + *pOutput = (double)pLeft[i] + pRight[0]; + } + } +} + +void vectorAdd(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, int32_t rightType, + void *output, int32_t order) { + switch(leftType) { + case TSDB_DATA_TYPE_TINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, int8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, uint8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, int16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, uint16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_INT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, int32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, uint32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, int64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, uint64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, float, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_ADD(left, numLeft, leftType, double, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + default:; + } +} + +void vectorSub(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, int32_t rightType, + void *output, int32_t order) { + switch(leftType) { + case TSDB_DATA_TYPE_TINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, int8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, uint8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, int16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, uint16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_INT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, int32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, uint32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, int64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, uint64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, float, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_SUB(left, numLeft, leftType, double, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + default:; + } +} + +void vectorMultiply(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, int32_t rightType, + void *output, int32_t order) { + switch(leftType) { + case TSDB_DATA_TYPE_TINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_INT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, int64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, uint64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, float, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_MULTIPLY(left, numLeft, leftType, double, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + default:; + } +} + +void vectorDivide(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, int32_t rightType, + void *output, int32_t order) { + switch(leftType) { + case TSDB_DATA_TYPE_TINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_INT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, int64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, uint64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, float, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_DIVIDE(left, numLeft, leftType, double, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + default:; + } +} + +void vectorRemainder(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, int32_t rightType, + void *output, int32_t order) { + switch(leftType) { + case TSDB_DATA_TYPE_TINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint8_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint16_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_INT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint32_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, int64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, uint64_t, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, float, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + switch (rightType) { + case TSDB_DATA_TYPE_TINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, int8_t, output, order); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, uint8_t, output, order); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, int16_t, output, order); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, uint16_t, output, order); + break; + } + case TSDB_DATA_TYPE_INT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, int32_t, output, order); + break; + } + case TSDB_DATA_TYPE_UINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, uint32_t, output, order); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, int64_t, output, order); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, uint64_t, output, order); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, float, output, order); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + DO_VECTOR_REMAINDER(left, numLeft, leftType, double, right, numRight, rightType, double, output, order); + break; + } + default: + assert(0); + } + break; + } + default:; + } +} + +_arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr) { + switch (arithmeticOptr) { + case TSDB_BINARY_OP_ADD: + return vectorAdd; + case TSDB_BINARY_OP_SUBTRACT: + return vectorSub; + case TSDB_BINARY_OP_MULTIPLY: + return vectorMultiply; + case TSDB_BINARY_OP_DIVIDE: + return vectorDivide; + case TSDB_BINARY_OP_REMAINDER: + return vectorRemainder; + default: + return NULL; + } +} diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index f21205479396ba606a1212f30350df2e0b3f59b5..f5b84e4c9ad7492ecea11eb232b33f373d37235a 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -289,23 +289,31 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { return NULL; } - pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); - if (pCols->cols == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); - tdFreeDataCols(pCols); - return NULL; + pCols->maxPoints = maxRows; + + if (maxCols > 0) { + pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); + if (pCols->cols == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, + strerror(errno)); + tdFreeDataCols(pCols); + return NULL; + } + + pCols->maxCols = maxCols; } pCols->maxRowSize = maxRowSize; - pCols->maxCols = maxCols; - pCols->maxPoints = maxRows; pCols->bufSize = maxRowSize * maxRows; - pCols->buf = malloc(pCols->bufSize); - if (pCols->buf == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); - tdFreeDataCols(pCols); - return NULL; + if (pCols->bufSize > 0) { + pCols->buf = malloc(pCols->bufSize); + if (pCols->buf == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, + strerror(errno)); + tdFreeDataCols(pCols); + return NULL; + } } return pCols; @@ -337,12 +345,13 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { return 0; } -void tdFreeDataCols(SDataCols *pCols) { +SDataCols *tdFreeDataCols(SDataCols *pCols) { if (pCols) { tfree(pCols->buf); tfree(pCols->cols); free(pCols); } + return NULL; } SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { diff --git a/src/query/src/qAst.c b/src/common/src/texpr.c similarity index 50% rename from src/query/src/qAst.c rename to src/common/src/texpr.c index a65f4a6dc95a586896372a648dc19ad2bb88b154..f50b829baa7c6dadea99bd81874cef57140f4fc0 100644 --- a/src/query/src/qAst.c +++ b/src/common/src/texpr.c @@ -16,29 +16,15 @@ #include "os.h" #include "exception.h" -#include "qAst.h" -#include "qSyntaxtreefunction.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" -#include "tname.h" #include "tsdb.h" #include "tskiplist.h" -#include "tsqlfunction.h" -#include "tstoken.h" -#include "tschemautil.h" - -typedef struct { - char* v; - int32_t optr; -} SEndPoint; - -typedef struct { - SEndPoint* start; - SEndPoint* end; -} SQueryCond; +#include "texpr.h" +#include "tarithoperator.h" static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { if (pLeft->nodeType == TSQL_NODE_COL) { @@ -53,323 +39,6 @@ static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, co } } -void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { - if (pNode == NULL) { - return; - } - - if (pNode->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&pNode, fp); - } else if (pNode->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy(pNode->pVal); - } else if (pNode->nodeType == TSQL_NODE_COL) { - free(pNode->pSchema); - } - - free(pNode); -} - -void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { - if (*pExpr == NULL) { - return; - } - - if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); - tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); - - if (fp != NULL) { - fp((*pExpr)->_node.info); - } - } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy((*pExpr)->pVal); - free((*pExpr)->pVal); - } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { - free((*pExpr)->pSchema); - } - - free(*pExpr); - *pExpr = NULL; -} - -// todo check for malloc failure -static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { - int32_t optr = queryColInfo->optr; - - if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || - optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(SEndPoint)); - pCond->start->optr = queryColInfo->optr; - pCond->start->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(SEndPoint)); - pCond->end->optr = queryColInfo->optr; - pCond->end->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - -static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { - SSkipListIterator* iter = NULL; - - SQueryCond cond = {0}; - if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { - //todo handle error - } - - if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); - } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); - } - - if (cond.start != NULL) { - int32_t optr = cond.start->optr; - - if (optr == TSDB_RELATION_EQUAL) { // equals - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - if (ret != 0) { - break; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal - bool comp = true; - int32_t ret = 0; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - assert(ret >= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_GREATER) { - continue; - } else { - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; - } - } - } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal - bool comp = true; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - tSkipListDestroyIter(iter); - - comp = true; - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - } else { - assert(0); - } - } else { - int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; - if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - bool comp = true; - int32_t ret = 0; - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); - assert(ret <= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_LESS) { - continue; - } else { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; // no need to compare anymore - } - } - } else { - assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); - if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || - (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } - } - } - - free(cond.start); - free(cond.end); - tSkipListDestroyIter(iter); -} - -static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - //non-leaf nodes, recursively traverse the expression tree in the post-root order - if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { - if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - if (filterItem(pLeft, pItem, param)) { - return true; - } - - // left child does not satisfy the query condition, try right child - return filterItem(pRight, pItem, param); - } else { // and - if (!filterItem(pLeft, pItem, param)) { - return false; - } - - return filterItem(pRight, pItem, param); - } - } - - // handle the leaf node - param->setupInfoFn(pExpr, param->pExtInfo); - return param->nodeFilterFn(pItem, pExpr->_node.info); -} - -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { - taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); - } - } - tSkipListDestroyIter(iter); -} - -static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - bool addToResult = false; - - SSkipListNode *pNode = tSkipListIterGet(iter); - char * pData = SL_GET_NODE_DATA(pNode); - - tstr *name = (tstr*) tsdbGetTableName((void*) pData); - - // todo speed up by using hash - if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { - if (pQueryInfo->optr == TSDB_RELATION_IN) { - addToResult = pQueryInfo->compare(name, pQueryInfo->q); - } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { - addToResult = !pQueryInfo->compare(name, pQueryInfo->q); - } - } else { - addToResult = filterFp(pNode, pQueryInfo); - } - - if (addToResult) { - STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(res, &info); - } - } - - tSkipListDestroyIter(iter); -} - -// post-root order traverse syntax tree -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { - if (pExpr == NULL) { - return; - } - - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - // column project - if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { - assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); - - param->setupInfoFn(pExpr, param->pExtInfo); - - tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { - tQueryIndexColumn(pSkipList, pQueryInfo, result); - } else { - tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); - } - - return; - } - - // The value of hasPK is always 0. - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; - assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); - - //apply the hierarchical expression to every node in skiplist for find the qualified nodes - tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - -#if 0 - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } - - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); - - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); -#endif -} - static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: { @@ -430,7 +99,75 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf } } -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); + +void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { + if (pNode == NULL) { + return; + } + + if (pNode->nodeType == TSQL_NODE_EXPR) { + doExprTreeDestroy(&pNode, fp); + } else if (pNode->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy(pNode->pVal); + } else if (pNode->nodeType == TSQL_NODE_COL) { + free(pNode->pSchema); + } + + free(pNode); +} + +static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { + if (*pExpr == NULL) { + return; + } + + if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { + doExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); + doExprTreeDestroy(&(*pExpr)->_node.pRight, fp); + + if (fp != NULL) { + fp((*pExpr)->_node.info); + } + } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy((*pExpr)->pVal); + free((*pExpr)->pVal); + } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { + free((*pExpr)->pSchema); + } + + free(*pExpr); + *pExpr = NULL; +} + +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + //non-leaf nodes, recursively traverse the expression tree in the post-root order + if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { + if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (exprTreeApplayFilter(pLeft, pItem, param)) { + return true; + } + + // left child does not satisfy the query condition, try right child + return exprTreeApplayFilter(pRight, pItem, param); + } else { // and + if (!exprTreeApplayFilter(pLeft, pItem, param)) { + return false; + } + + return exprTreeApplayFilter(pRight, pItem, param); + } + } + + // handle the leaf node + param->setupInfoFn(pExpr, param->pExtInfo); + return param->nodeFilterFn(pItem, pExpr->_node.info); +} + +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { return; @@ -442,7 +179,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, /* the left output has result from the left child syntax tree */ char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } /* the right output has result from the right child syntax tree */ @@ -450,7 +187,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, char *pdata = malloc(sizeof(int64_t) * numOfRows); if (pRight->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } if (pLeft->nodeType == TSQL_NODE_EXPR) { @@ -459,81 +196,82 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, * exprLeft + exprRight * the type of returned value of one expression is always double float precious */ - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); - fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); + OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); // set input buffer char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pInputData, pRight->pSchema->type, numOfRows); - fp(pLeftOutput, pdata, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pdata, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); } else { - fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); - fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); + OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); } } else if (pLeft->nodeType == TSQL_NODE_COL) { // column data specified on left-hand-side char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId); if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); - fp(pdata, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + OperatorFn(pdata, numOfRows, pLeft->pSchema->type, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); } else { - fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); } } else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); // both columns are descending order, do not reverse the source data - fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order); + OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); - fp(pdata, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); + OperatorFn(pdata, numOfRows, pLeft->pSchema->type, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); } else { - fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); + OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); } } } else { // column data specified on left-hand-side if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); - fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, TSDB_ORDER_ASC); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); + OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows); - fp(&pLeft->pVal->i64Key, pdata, numOfRows, 1, pOutput, TSDB_ORDER_ASC); + OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, pdata, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); } else { - fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, TSDB_ORDER_ASC); + OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); - fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, TSDB_ORDER_ASC); + _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); + OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); } } - free(pLeftOutput); - free(pRightOutput); + tfree(pdata); + tfree(pLeftOutput); + tfree(pRightOutput); } static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { @@ -547,7 +285,7 @@ static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { tbufWriteInt32(bw, pVal->nLen); tbufWrite(bw, pVal->pz, pVal->nLen); } else { - tbufWriteInt64(bw, pVal->i64Key); + tbufWriteInt64(bw, pVal->i64); } } else if (expr->nodeType == TSQL_NODE_COL) { @@ -604,7 +342,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { } tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode)); - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, pExpr, NULL); pExpr->nodeType = tbufReadUint8(br); if (pExpr->nodeType == TSQL_NODE_VALUE) { @@ -617,7 +355,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { pVal->pz = calloc(1, pVal->nLen + 1); tbufReadToBuffer(br, pVal->pz, pVal->nLen); } else { - pVal->i64Key = tbufReadInt64(br); + pVal->i64 = tbufReadInt64(br); } } else if (pExpr->nodeType == TSQL_NODE_COL) { @@ -658,7 +396,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { int32_t anchor = CLEANUP_GET_ANCHOR(); tExprNode* expr = exception_calloc(1, sizeof(tExprNode)); - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL); expr->nodeType = TSQL_NODE_EXPR; @@ -669,7 +407,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { SSchema* pSchema = exception_calloc(1, sizeof(SSchema)); left->pSchema = pSchema; - *pSchema = tscGetTbnameColumnSchema(); + *pSchema = *tGetTbnameColumnSchema(); tExprNode* right = exception_calloc(1, sizeof(tExprNode)); expr->_node.pRight = right; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index a912cdfd7f531a6d863323056b1d9f78857a9284..4ed4e0473bdf3c11c2838842b24c15d886a02f2e 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -25,7 +25,6 @@ #include "tutil.h" #include "tlocale.h" #include "ttimezone.h" -#include "tsync.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -40,26 +39,26 @@ uint16_t tsSyncPort = 6040; uint16_t tsArbitratorPort = 6042; int32_t tsStatusInterval = 1; // second int32_t tsNumOfMnodes = 3; -int32_t tsEnableVnodeBak = 1; -int32_t tsEnableTelemetryReporting = 1; +int8_t tsEnableVnodeBak = 1; +int8_t tsEnableTelemetryReporting = 1; +int8_t tsArbOnline = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; // common int32_t tsRpcTimer = 1000; int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsMaxShellConns = 5000; +int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 1; -float tsRatioOfQueryThreads = 0.5f; +float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string -int32_t tsEnableCoreFile = 0; +int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; -char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -72,9 +71,8 @@ char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; int32_t tsCompressMsgSize = -1; // client -int32_t tsTableMetaKeepTimer = 7200; // second int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; -int32_t tsTscEnableRecordSql = 0; +int8_t tsTscEnableRecordSql = 0; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -106,9 +104,13 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance // 0 no query allowed, queries are disabled // positive value (in MB) int32_t tsQueryBufferSize = -1; +int64_t tsQueryBufferSizeBytes = -1; -// only 50% cpu will be used in query processing in dnode -int32_t tsHalfCoresForQuery = 0; +// in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. +int32_t tsRetrieveBlockingModel = 0; + +// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name +int8_t tsKeepOriginalColumnName = 0; // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; @@ -119,38 +121,41 @@ int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; -int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; -int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; +int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; +int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; +int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; // balance -int32_t tsEnableBalance = 1; -int32_t tsAlternativeRole = 0; -int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400*100; // seconds 10days +int8_t tsEnableBalance = 1; +int8_t tsAlternativeRole = 0; +int32_t tsBalanceInterval = 300; // seconds +int32_t tsOfflineThreshold = 86400 * 100; // seconds 100 days int32_t tsMnodeEqualVnodeNum = 4; -int32_t tsFlowCtrl = 1; +int8_t tsEnableFlowCtrl = 1; +int8_t tsEnableSlaveQuery = 1; +int8_t tsEnableAdjustMaster = 1; // restful -int32_t tsEnableHttpModule = 1; +int8_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] int32_t tsHttpCacheSessions = 1000; int32_t tsHttpSessionExpire = 36000; int32_t tsHttpMaxThreads = 2; -int32_t tsHttpEnableCompress = 1; -int32_t tsHttpEnableRecordSql = 0; -int32_t tsTelegrafUseFieldNum = 0; +int8_t tsHttpEnableCompress = 1; +int8_t tsHttpEnableRecordSql = 0; +int8_t tsTelegrafUseFieldNum = 0; // mqtt -int32_t tsEnableMqttModule = 0; // not finished yet, not started it by default +int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org"; char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883"; char tsMqttUser[TSDB_MQTT_USER_LEN] = {0}; @@ -159,24 +164,32 @@ char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber"; char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // # // monitor -int32_t tsEnableMonitorModule = 1; +int8_t tsEnableMonitorModule = 1; char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsInternalPass[] = "secretkey"; int32_t tsMonitorInterval = 30; // seconds // stream -int32_t tsEnableStream = 1; +int8_t tsEnableStream = 1; // internal -int32_t tsPrintAuth = 0; -int32_t tscEmbedded = 0; -char configDir[TSDB_FILENAME_LEN] = {0}; -char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsDataDir[TSDB_FILENAME_LEN] = {0}; -char tsScriptDir[TSDB_FILENAME_LEN] = {0}; -char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; +int8_t tsPrintAuth = 0; +int8_t tscEmbedded = 0; +char configDir[TSDB_FILENAME_LEN] = {0}; +char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsDataDir[TSDB_FILENAME_LEN] = {0}; +char tsScriptDir[TSDB_FILENAME_LEN] = {0}; +char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; + +int32_t tsDiskCfgNum = 0; + +#ifndef _STORAGE +SDiskCfg tsDiskCfg[1]; +#else +SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; +#endif /* * minimum scale for whole system, millisecond by default @@ -196,31 +209,32 @@ float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; float tsAvailTmpDirectorySpace = 0; float tsAvailDataDirGB = 0; -float tsReservedTmpDirectorySpace = 0.1f; -float tsMinimalDataDirGB = 0.5f; +float tsReservedTmpDirectorySpace = 1.0f; +float tsMinimalDataDirGB = 1.0f; int32_t tsTotalMemoryMB = 0; -int32_t tsVersion = 0; +uint32_t tsVersion = 0; // log int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 135; -int32_t sdbDebugFlag = 135; +int32_t mDebugFlag = 131; +int32_t sdbDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; -int32_t cDebugFlag = 131; +uint32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; int32_t httpDebugFlag = 131; int32_t mqttDebugFlag = 131; int32_t monDebugFlag = 131; -int32_t qDebugFlag = 131; +uint32_t qDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; -int32_t cqDebugFlag = 135; +int32_t cqDebugFlag = 131; +int32_t fsDebugFlag = 135; int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; @@ -259,7 +273,7 @@ bool taosCfgDynamicOptions(char *msg) { int32_t vint = 0; paGetToken(msg, &option, &olen); - if (olen == 0) return TSDB_CODE_COM_INVALID_CFG_MSG; + if (olen == 0) return false;; paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) @@ -273,12 +287,16 @@ bool taosCfgDynamicOptions(char *msg) { for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (cfg->valType != TAOS_CFG_VTYPE_INT32) continue; + if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; - *((int32_t *)cfg->ptr) = vint; + if (cfg->valType == TAOS_CFG_VTYPE_INT32) { + *((int32_t *)cfg->ptr) = vint; + } else { + *((int8_t *)cfg->ptr) = (int8_t)vint; + } if (strncasecmp(cfg->option, "monitor", olen) == 0) { if (1 == vint) { @@ -298,11 +316,9 @@ bool taosCfgDynamicOptions(char *msg) { } return true; } - if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); + taosSetAllDebugFlag(); } - return true; } @@ -324,6 +340,56 @@ bool taosCfgDynamicOptions(char *msg) { return false; } +void taosAddDataDir(int index, char *v1, int level, int primary) { + tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); + tsDiskCfg[index].level = level; + tsDiskCfg[index].primary = primary; + uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); +} + +#ifndef _STORAGE +void taosReadDataDirCfg(char *v1, char *v2, char *v3) { + if (tsDiskCfgNum == 1) { + SDiskCfg *cfg = &tsDiskCfg[0]; + uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); + } + taosAddDataDir(0, v1, 0, 1); + tsDiskCfgNum = 1; +} + +void taosPrintDataDirCfg() { + for (int i = 0; i < tsDiskCfgNum; ++i) { + SDiskCfg *cfg = &tsDiskCfg[i]; + uInfo(" dataDir: %s", cfg->dir); + } +} +#endif + +static void taosCheckDataDirCfg() { + if (tsDiskCfgNum <= 0) { + taosAddDataDir(0, tsDataDir, 0, 1); + tsDiskCfgNum = 1; + uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); + } +} + +static int32_t taosCheckTmpDir(void) { + if (strlen(tsTempDir) <= 0){ + uError("tempDir is not set"); + return -1; + } + + DIR *dir = opendir(tsTempDir); + if (dir == NULL) { + uError("can not open tempDir:%s, error:%s", tsTempDir, strerror(errno)); + return -1; + } + + closedir(dir); + + return 0; +} + static void doInitGlobalConfig(void) { osInit(); srand(taosSafeRand()); @@ -364,10 +430,10 @@ static void doInitGlobalConfig(void) { // port cfg.option = "serverPort"; cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_UINT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 1; - cfg.maxValue = 65535; + cfg.maxValue = 65056; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -405,7 +471,7 @@ static void doInitGlobalConfig(void) { cfg.option = "dataDir"; cfg.ptr = tsDataDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; + cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 0; @@ -416,7 +482,7 @@ static void doInitGlobalConfig(void) { cfg.option = "arbitrator"; cfg.ptr = tsArbitrator; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; @@ -444,12 +510,12 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "ratioOfQueryThreads"; - cfg.ptr = &tsRatioOfQueryThreads; + cfg.option = "ratioOfQueryCores"; + cfg.ptr = &tsRatioOfQueryCores; cfg.valType = TAOS_CFG_VTYPE_FLOAT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0.1f; - cfg.maxValue = 0.9f; + cfg.minValue = 0.0f; + cfg.maxValue = 2.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -466,7 +532,7 @@ static void doInitGlobalConfig(void) { cfg.option = "vnodeBak"; cfg.ptr = &tsEnableVnodeBak; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -476,7 +542,7 @@ static void doInitGlobalConfig(void) { cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -486,7 +552,7 @@ static void doInitGlobalConfig(void) { cfg.option = "balance"; cfg.ptr = &tsEnableBalance; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -507,7 +573,7 @@ static void doInitGlobalConfig(void) { // 0-any; 1-mnode; 2-vnode cfg.option = "role"; cfg.ptr = &tsAlternativeRole; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 2; @@ -540,8 +606,8 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsOfflineThreshold; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 5; - cfg.maxValue = 7200000; + cfg.minValue = 3; + cfg.maxValue = 86400 * 365; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); @@ -586,16 +652,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); - cfg.option = "tableMetaKeepTimer"; - cfg.ptr = &tsTableMetaKeepTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 8640000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); - cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -759,7 +815,7 @@ static void doInitGlobalConfig(void) { cfg.option = "comp"; cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_COMP_LEVEL; cfg.maxValue = TSDB_MAX_COMP_LEVEL; @@ -769,7 +825,7 @@ static void doInitGlobalConfig(void) { cfg.option = "walLevel"; cfg.ptr = &tsWAL; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_WAL_LEVEL; cfg.maxValue = TSDB_MAX_WAL_LEVEL; @@ -801,15 +857,15 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsQuorum; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION; - cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; + cfg.minValue = TSDB_MIN_DB_QUORUM_OPTION; + cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); cfg.option = "update"; cfg.ptr = &tsUpdate; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_DB_UPDATE; cfg.maxValue = TSDB_MAX_DB_UPDATE; @@ -887,8 +943,8 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); - cfg.option = "halfCoresForQuery"; - cfg.ptr = &tsHalfCoresForQuery; + cfg.option = "retrieveBlockingModel"; + cfg.ptr = &tsRetrieveBlockingModel; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; @@ -897,11 +953,21 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "keepColumnName"; + cfg.ptr = &tsKeepOriginalColumnName; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + // locale & charset cfg.option = "timezone"; cfg.ptr = tsTimezone; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = tListLen(tsTimezone); @@ -911,7 +977,7 @@ static void doInitGlobalConfig(void) { cfg.option = "locale"; cfg.ptr = tsLocale; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = tListLen(tsLocale); @@ -921,7 +987,7 @@ static void doInitGlobalConfig(void) { cfg.option = "charset"; cfg.ptr = tsCharset; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = tListLen(tsCharset); @@ -992,8 +1058,28 @@ static void doInitGlobalConfig(void) { // module configs cfg.option = "flowctrl"; - cfg.ptr = &tsFlowCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.ptr = &tsEnableFlowCtrl; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "slaveQuery"; + cfg.ptr = &tsEnableSlaveQuery; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "adjustMaster"; + cfg.ptr = &tsEnableAdjustMaster; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1003,7 +1089,7 @@ static void doInitGlobalConfig(void) { cfg.option = "http"; cfg.ptr = &tsEnableHttpModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1013,7 +1099,7 @@ static void doInitGlobalConfig(void) { cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1023,7 +1109,7 @@ static void doInitGlobalConfig(void) { cfg.option = "monitor"; cfg.ptr = &tsEnableMonitorModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1033,7 +1119,7 @@ static void doInitGlobalConfig(void) { cfg.option = "stream"; cfg.ptr = &tsEnableStream; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1043,7 +1129,7 @@ static void doInitGlobalConfig(void) { cfg.option = "httpEnableRecordSql"; cfg.ptr = &tsHttpEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1053,7 +1139,7 @@ static void doInitGlobalConfig(void) { cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1065,7 +1151,7 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsHttpMaxThreads; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; + cfg.minValue = 2; cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; @@ -1104,7 +1190,7 @@ static void doInitGlobalConfig(void) { cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 1; @@ -1305,7 +1391,7 @@ static void doInitGlobalConfig(void) { cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1315,7 +1401,7 @@ static void doInitGlobalConfig(void) { cfg.option = "enableCoreFile"; cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1418,6 +1504,12 @@ int32_t taosCheckGlobalCfg() { snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port); } + taosCheckDataDirCfg(); + + if (taosCheckTmpDir()) { + return -1; + } + taosGetSystemInfo(); tsSetLocale(); @@ -1431,23 +1523,47 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } + if (tsHttpMaxThreads == 2) { + int32_t halfNumOfCores = tsNumOfCores >> 1; + if (halfNumOfCores > 2) { + tsHttpMaxThreads = halfNumOfCores; + } + } + + if (tsMaxTablePerVnode < tsMinTablePerVnode) { + uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", + tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode); + tsMaxTablePerVnode = tsMinTablePerVnode; + } + // todo refactor tsVersion = 0; - for (int i = 0; i < 10; i++) { + for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { if (version[i] >= '0' && version[i] <= '9') { - tsVersion = tsVersion * 10 + (version[i] - '0'); + ver = ver * 10 + (version[i] - '0'); + } else if (version[i] == '.') { + tsVersion |= ver & 0xFF; + tsVersion <<= 8; + + ver = 0; } else if (version[i] == 0) { + tsVersion |= ver & 0xFF; + break; } } - - tsVersion = 10 * tsVersion; tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035] tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE; // udp/tcp tsSyncPort = tsServerPort + TSDB_PORT_SYNC; tsHttpPort = tsServerPort + TSDB_PORT_HTTP; + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } + + uInfo(" check global cfg completed"); + uInfo("=================================="); taosPrintGlobalCfg(); return 0; diff --git a/src/common/src/tname.c b/src/common/src/tname.c index bea8c52ef21ea607bee5379ace879ef936ff60bb..787aa1e95b0b361d1fe4fc6931e7da04aa419c65 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -3,28 +3,14 @@ #include "tname.h" #include "tstoken.h" -#include "ttokendef.h" #include "tvariant.h" -// todo refactor -UNUSED_FUNC static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { - for (int32_t i = 0; i < num; ++i) { - while (*input != 0 && *input++ != delim) { - }; - } - return input; -} +#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) +#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) -UNUSED_FUNC static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { - size_t len = 0; - while (*src != delimiter && *src != 0) { - *dst++ = *src++; - len++; - } - - return len; -} +#define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) +//TODO remove it void extractTableName(const char* tableId, char* name) { size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]); size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]); @@ -39,12 +25,20 @@ char* extractDBName(const char* tableId, char* name) { return strncpy(name, &tableId[offset1 + 1], len); } -SSchema tGetTableNameColumnSchema() { +// todo remove it +size_t tableIdPrefix(const char* name, char* prefix, int32_t len) { + tstrncpy(prefix, name, len); + strcat(prefix, TS_PATH_DELIMITER); + + return strlen(prefix); +} + +SSchema tGetBlockDistColumnSchema() { SSchema s = {0}; - s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; + s.bytes = TSDB_MAX_BINARY_LEN;; s.type = TSDB_DATA_TYPE_BINARY; - s.colId = TSDB_TBNAME_COLUMN_INDEX; - tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + s.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; + tstrncpy(s.name, TSQL_BLOCK_DIST_L, TSDB_COL_NAME_LEN); return s; } @@ -55,7 +49,7 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); } else { - s.bytes = tDataTypeDesc[pVal->nType].nSize; + s.bytes = tDataTypes[pVal->nType].bytes; } s.colId = TSDB_UD_COLUMN_INDEX; @@ -74,7 +68,7 @@ bool tscValidateTableNameLength(size_t len) { return len < TSDB_TABLE_NAME_LEN; } -SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { +SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) { if (numOfFilters == 0) { assert(src == NULL); return NULL; @@ -189,13 +183,256 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) { } } -SSchema tscGetTbnameColumnSchema() { - struct SSchema s = { - .colId = TSDB_TBNAME_COLUMN_INDEX, - .type = TSDB_DATA_TYPE_BINARY, - .bytes = TSDB_TABLE_NAME_LEN - }; +static struct SSchema _s = { + .colId = TSDB_TBNAME_COLUMN_INDEX, + .type = TSDB_DATA_TYPE_BINARY, + .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, + .name = TSQL_TBNAME_L, +}; - strcpy(s.name, TSQL_TBNAME_L); - return s; -} \ No newline at end of file +SSchema* tGetTbnameColumnSchema() { + return &_s; +} + +static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { + int32_t rowLen = 0; + + for (int32_t i = 0; i < numOfCols; ++i) { + // 1. valid types + if (!isValidDataType(pSchema[i].type)) { + return false; + } + + // 2. valid length for each type + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { + if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { + return false; + } + } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { + return false; + } + } else { + if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) { + return false; + } + } + + // 3. valid column names + for (int32_t j = i + 1; j < numOfCols; ++j) { + if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { + return false; + } + } + + rowLen += pSchema[i].bytes; + } + + return rowLen <= maxLen; +} + +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) { + if (!VALIDNUMOFCOLS(numOfCols)) { + return false; + } + + if (!VALIDNUMOFTAGS(numOfTags)) { + return false; + } + + /* first column must be the timestamp, which is a primary key */ + if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { + return false; + } + + if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) { + return false; + } + + if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) { + return false; + } + + return true; +} + +int32_t tNameExtractFullName(const SName* name, char* dst) { + assert(name != NULL && dst != NULL); + + // invalid full name format, abort + if (!tIsValidName(name)) { + return -1; + } + + int32_t len = snprintf(dst, TSDB_ACCT_ID_LEN + 1 + TSDB_DB_NAME_LEN, "%s.%s", name->acctId, name->dbname); + + size_t tnameLen = strlen(name->tname); + if (tnameLen > 0) { + assert(name->type == TSDB_TABLE_NAME_T); + dst[len] = TS_PATH_DELIMITER[0]; + + memcpy(dst + len + 1, name->tname, tnameLen); + dst[len + tnameLen + 1] = 0; + } + + return 0; +} + +int32_t tNameLen(const SName* name) { + assert(name != NULL); + int32_t len = (int32_t) strlen(name->acctId); + int32_t len1 = (int32_t) strlen(name->dbname); + int32_t len2 = (int32_t) strlen(name->tname); + + if (name->type == TSDB_DB_NAME_T) { + assert(len2 == 0); + return len + len1 + TS_PATH_DELIMITER_LEN; + } else { + assert(len2 > 0); + return len + len1 + len2 + TS_PATH_DELIMITER_LEN * 2; + } +} + +bool tIsValidName(const SName* name) { + assert(name != NULL); + + if (!VALID_NAME_TYPE(name->type)) { + return false; + } + + if (strlen(name->acctId) <= 0) { + return false; + } + + if (name->type == TSDB_DB_NAME_T) { + return strlen(name->dbname) > 0; + } else { + return strlen(name->dbname) > 0 && strlen(name->tname) > 0; + } +} + +SName* tNameDup(const SName* name) { + assert(name != NULL); + + SName* p = calloc(1, sizeof(SName)); + memcpy(p, name, sizeof(SName)); + return p; +} + +int32_t tNameGetDbName(const SName* name, char* dst) { + assert(name != NULL && dst != NULL); + strncpy(dst, name->dbname, tListLen(name->dbname)); + return 0; +} + +int32_t tNameGetFullDbName(const SName* name, char* dst) { + assert(name != NULL && dst != NULL); + snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, + "%s.%s", name->acctId, name->dbname); + return 0; +} + +bool tNameIsEmpty(const SName* name) { + assert(name != NULL); + return name->type == 0 || strlen(name->acctId) <= 0; +} + +const char* tNameGetTableName(const SName* name) { + assert(name != NULL && name->type == TSDB_TABLE_NAME_T); + return &name->tname[0]; +} + +void tNameAssign(SName* dst, const SName* src) { + memcpy(dst, src, sizeof(SName)); +} + +int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken) { + assert(dst != NULL && dbToken != NULL && acct != NULL); + + // too long account id or too long db name + if (strlen(acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) { + return -1; + } + + dst->type = TSDB_DB_NAME_T; + tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + tstrncpy(dst->dbname, dbToken->z, dbToken->n + 1); + return 0; +} + +int32_t tNameSetAcctId(SName* dst, const char* acct) { + assert(dst != NULL && acct != NULL); + + // too long account id or too long db name + if (strlen(acct) >= tListLen(dst->acctId)) { + return -1; + } + + tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + + assert(strlen(dst->acctId) > 0); + + return 0; +} + +int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { + assert(dst != NULL && str != NULL && strlen(str) > 0); + + char* p = NULL; + if ((type & T_NAME_ACCT) == T_NAME_ACCT) { + p = strstr(str, TS_PATH_DELIMITER); + if (p == NULL) { + return -1; + } + + int32_t len = (int32_t)(p - str); + + // too long account id or too long db name + if ((len >= tListLen(dst->acctId)) || (len <= 0)) { + return -1; + } + + memcpy (dst->acctId, str, len); + dst->acctId[len] = 0; + + assert(strlen(dst->acctId) > 0); + } + + if ((type & T_NAME_DB) == T_NAME_DB) { + dst->type = TSDB_DB_NAME_T; + char* start = (char*)((p == NULL)? str:(p+1)); + + int32_t len = 0; + p = strstr(start, TS_PATH_DELIMITER); + if (p == NULL) { + len = (int32_t) strlen(start); + } else { + len = (int32_t) (p - start); + } + + // too long account id or too long db name + if ((len >= tListLen(dst->dbname)) || (len <= 0)) { + return -1; + } + + memcpy (dst->dbname, start, len); + dst->dbname[len] = 0; + } + + if ((type & T_NAME_TABLE) == T_NAME_TABLE) { + dst->type = TSDB_TABLE_NAME_T; + char* start = (char*) ((p == NULL)? str: (p+1)); + + int32_t len = (int32_t) strlen(start); + + // too long account id or too long db name + if ((len >= tListLen(dst->tname)) || (len <= 0)) { + return -1; + } + + memcpy (dst->tname, start, len); + dst->tname[len] = 0; + } + + return 0; +} diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index f28481977f41bf550205d2669c6f90a016d2b7c4..6fa27a029bfd5356cca3e34dffe8d3018ade9fd8 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -14,11 +14,11 @@ */ #include "os.h" -#include "taosdef.h" +#include "ttype.h" #include "ttokendef.h" #include "tscompression.h" -const int32_t TYPE_BYTES[11] = { +const int32_t TYPE_BYTES[15] = { -1, // TSDB_DATA_TYPE_NULL sizeof(int8_t), // TSDB_DATA_TYPE_BOOL sizeof(int8_t), // TSDB_DATA_TYPE_TINYINT @@ -29,10 +29,28 @@ const int32_t TYPE_BYTES[11] = { sizeof(double), // TSDB_DATA_TYPE_DOUBLE sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_BINARY sizeof(TSKEY), // TSDB_DATA_TYPE_TIMESTAMP - sizeof(VarDataOffsetT) // TSDB_DATA_TYPE_NCHAR + sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_NCHAR + sizeof(uint8_t), // TSDB_DATA_TYPE_UTINYINT + sizeof(uint16_t), // TSDB_DATA_TYPE_USMALLINT + sizeof(uint32_t), // TSDB_DATA_TYPE_UINT + sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT }; -static void getStatics_bool(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, +#define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \ + do { \ + (__sum) += (_list)[(_index)]; \ + if ((__min) > (_list)[(_index)]) { \ + (__min) = (_list)[(_index)]; \ + (__minIndex) = (_index); \ + } \ + \ + if ((__max) < (_list)[(_index)]) { \ + (__max) = (_list)[(_index)]; \ + (__maxIndex) = (_index); \ + } \ + } while (0) + +static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; @@ -43,26 +61,17 @@ static void getStatics_bool(const TSKEY *primaryKey, const void *pData, int32_t ASSERT(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((char *)&data[i], TSDB_DATA_TYPE_BOOL)) { + if (data[i] == TSDB_DATA_BOOL_NULL) { (*numOfNull) += 1; continue; } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; - } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); } } -static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -72,26 +81,43 @@ static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t nu ASSERT(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((char *)&data[i], TSDB_DATA_TYPE_TINYINT)) { + if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { (*numOfNull) += 1; continue; } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint8_t *data = (uint8_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { + (*numOfNull) += 1; + continue; } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); } + + *min = _min; + *max = _max; + *sum = _sum; } -static void getStatics_i16(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int16_t *data = (int16_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -100,39 +126,45 @@ static void getStatics_i16(const TSKEY *primaryKey, const void *pData, int32_t n ASSERT(numOfRow <= INT16_MAX); - // int64_t lastKey = 0; - // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) { + if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { (*numOfNull) += 1; continue; } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } + +} + +static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint16_t *data = (uint16_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { + (*numOfNull) += 1; + continue; } - - // if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); } + + *min = _min; + *max = _max; + *sum = _sum; } -static void getStatics_i32(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int32_t *data = (int32_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -141,29 +173,43 @@ static void getStatics_i32(const TSKEY *primaryKey, const void *pData, int32_t n ASSERT(numOfRow <= INT16_MAX); - // int64_t lastKey = 0; - // int32_t lastVal = TSDB_DATA_INT_NULL; - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) { + if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { (*numOfNull) += 1; continue; } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint32_t *data = (uint32_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { + (*numOfNull) += 1; + continue; } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); } + + *min = _min; + *max = _max; + *sum = _sum; } -static void getStatics_i64(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, +static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *data = (int64_t *)pData; *min = INT64_MAX; @@ -174,52 +220,60 @@ static void getStatics_i64(const TSKEY *primaryKey, const void *pData, int32_t n ASSERT(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) { + if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { (*numOfNull) += 1; continue; } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint64_t *data = (uint64_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { + (*numOfNull) += 1; + continue; } - - // if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); } + + *min = _min; + *max = _max; + *sum = _sum; } -static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, +static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { - float *data = (float *)pData; - float fmin = FLT_MAX; - float fmax = -FLT_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; + float *data = (float *)pData; + float fmin = FLT_MAX; + float fmax = -FLT_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; ASSERT(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) { + if ((*(uint32_t*)&(data[i])) == TSDB_DATA_FLOAT_NULL) { (*numOfNull) += 1; continue; } - - float fv = 0; - fv = GET_FLOAT_VAL((const char*)&(data[i])); + + float fv = GET_FLOAT_VAL((const char*)&(data[i])); + dsum += fv; if (fmin > fv) { fmin = fv; @@ -232,28 +286,24 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num } } - double csum = 0; - csum = GET_DOUBLE_VAL((const char *)sum); - csum += dsum; - - SET_DOUBLE_VAL(sum, csum); + SET_DOUBLE_VAL(sum, dsum); SET_DOUBLE_VAL(max, fmax); SET_DOUBLE_VAL(min, fmin); } -static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, +static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { double *data = (double *)pData; - double dmin = DBL_MAX; - double dmax = -DBL_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; ASSERT(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) { + if ((*(uint64_t*)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { (*numOfNull) += 1; continue; } @@ -272,16 +322,12 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num } } - double csum = 0; - csum = GET_DOUBLE_VAL((const char *)sum); - csum += dsum; - - SET_DOUBLE_PTR(sum, &csum); + SET_DOUBLE_PTR(sum, &dsum); SET_DOUBLE_PTR(max, &dmax); SET_DOUBLE_PTR(min, &dmin); } -static void getStatics_bin(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, +static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char* data = pData; ASSERT(numOfRow <= INT16_MAX); @@ -301,7 +347,7 @@ static void getStatics_bin(const TSKEY *primaryKey, const void *pData, int32_t n *maxIndex = 0; } -static void getStatics_nchr(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, +static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char* data = pData; ASSERT(numOfRow <= INT16_MAX); @@ -321,18 +367,22 @@ static void getStatics_nchr(const TSKEY *primaryKey, const void *pData, int32_t *maxIndex = 0; } -tDataTypeDescriptor tDataTypeDesc[11] = { - {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", NULL, NULL, NULL}, - {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, getStatics_bool}, - {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, - {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, - {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt, getStatics_i32}, - {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint, getStatics_i64}, - {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat, getStatics_f}, - {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble, getStatics_d}, - {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, getStatics_bin}, - {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, - {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, getStatics_nchr}, +tDataTypeDescriptor tDataTypes[15] = { + {TSDB_DATA_TYPE_NULL, 6,1, "NOTYPE", NULL, NULL, NULL}, + {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, getStatics_bool}, + {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, + {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, + {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt, getStatics_i32}, + {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint, getStatics_i64}, + {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat, getStatics_f}, + {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble, getStatics_d}, + {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, + {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, + {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, + {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", tsCompressInt, tsDecompressInt, getStatics_u32}, + {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", tsCompressBigint, tsDecompressBigint, getStatics_u64}, }; char tTokenTypeSwitcher[13] = { @@ -352,7 +402,7 @@ char tTokenTypeSwitcher[13] = { }; bool isValidDataType(int32_t type) { - return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR; + return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; } void setVardataNull(char* val, int32_t type) { @@ -373,38 +423,58 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { switch (type) { case TSDB_DATA_TYPE_BOOL: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_BOOL_NULL; + *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_BOOL_NULL; } break; case TSDB_DATA_TYPE_TINYINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_TINYINT_NULL; + *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_TINYINT_NULL; } break; case TSDB_DATA_TYPE_SMALLINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint16_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_SMALLINT_NULL; + *(uint16_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_SMALLINT_NULL; } break; case TSDB_DATA_TYPE_INT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_INT_NULL; + *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_INT_NULL; } break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_BIGINT_NULL; + *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_BIGINT_NULL; + } + break; + case TSDB_DATA_TYPE_UTINYINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UTINYINT_NULL; + } + break; + case TSDB_DATA_TYPE_USMALLINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint16_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_USMALLINT_NULL; + } + break; + case TSDB_DATA_TYPE_UINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UINT_NULL; + } + break; + case TSDB_DATA_TYPE_UBIGINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UBIGINT_NULL; } break; case TSDB_DATA_TYPE_FLOAT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_FLOAT_NULL; + *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_FLOAT_NULL; } break; case TSDB_DATA_TYPE_DOUBLE: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL; + *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_DOUBLE_NULL; } break; case TSDB_DATA_TYPE_NCHAR: @@ -415,20 +485,24 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { break; default: { for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize) = TSDB_DATA_INT_NULL; + *(uint32_t *)(val + i * tDataTypes[TSDB_DATA_TYPE_INT].bytes) = TSDB_DATA_INT_NULL; } break; } } } -static uint8_t nullBool = TSDB_DATA_BOOL_NULL; -static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; -static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; -static uint32_t nullInt = TSDB_DATA_INT_NULL; -static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; -static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; -static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; +static uint8_t nullBool = TSDB_DATA_BOOL_NULL; +static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; +static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; +static uint32_t nullInt = TSDB_DATA_INT_NULL; +static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; +static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; +static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; +static uint8_t nullTinyIntu = TSDB_DATA_UTINYINT_NULL; +static uint16_t nullSmallIntu = TSDB_DATA_USMALLINT_NULL; +static uint32_t nullIntu = TSDB_DATA_UINT_NULL; +static uint64_t nullBigIntu = TSDB_DATA_UBIGINT_NULL; static union { tstr str; @@ -436,38 +510,43 @@ static union { } nullBinary = {.str = {.len = 1}}, nullNchar = {.str = {.len = 4}}; static void *nullValues[] = { - &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, - &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, + &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, + &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, + &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, }; void *getNullValue(int32_t type) { - assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_NCHAR); + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); return nullValues[type - 1]; } void assignVal(char *val, const char *src, int32_t len, int32_t type) { switch (type) { - case TSDB_DATA_TYPE_INT: { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *((int8_t *)val) = GET_INT8_VAL(src); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *((int16_t *)val) = GET_INT16_VAL(src); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: *((int32_t *)val) = GET_INT32_VAL(src); break; - } + case TSDB_DATA_TYPE_FLOAT: SET_FLOAT_VAL(val, GET_FLOAT_VAL(src)); break; case TSDB_DATA_TYPE_DOUBLE: SET_DOUBLE_VAL(val, GET_DOUBLE_VAL(src)); break; - case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: *((int64_t *)val) = GET_INT64_VAL(src); break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)val) = GET_INT16_VAL(src); - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)val) = GET_INT8_VAL(src); - break; case TSDB_DATA_TYPE_BINARY: varDataCopy(val, src); break; @@ -483,12 +562,14 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { - case TSDB_DATA_TYPE_INT: { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); break; } case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { SWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); break; @@ -497,7 +578,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf SWAP(*(double *)(pLeft), *(double *)(pRight), double); break; } - case TSDB_DATA_TYPE_SMALLINT: { + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { SWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); break; } @@ -508,7 +590,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf } case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { SWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); break; } @@ -521,3 +604,41 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf } } } + +int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) { + errno = 0; + int32_t ret = 0; + + char* endPtr = NULL; + if (type == TK_FLOAT) { + double v = strtod(z, &endPtr); + if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) { + ret = -1; + } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) { + ret = -1; + } else { + *value = (int64_t) round(v); + } + + errno = 0; + return ret; + } + + int32_t radix = 10; + if (type == TK_HEX) { + radix = 16; + } else if (type == TK_BIN) { + radix = 2; + } + + // the string may be overflow according to errno + *value = issigned? strtoll(z, &endPtr, radix):strtoull(z, &endPtr, radix); + + // not a valid integer number, return error + if (endPtr - z != n || errno == ERANGE) { + ret = -1; + } + + errno = 0; + return ret; +} diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index fc00f50a7af0cd6b92262438baa293dda7066a07..7009c4d5c3d8516f17dd4caf89b37e9c28f93274 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -21,36 +21,48 @@ #include "tstoken.h" #include "ttokendef.h" #include "tutil.h" +#include "ttype.h" -// todo support scientific expression number and oct number -void tVariantCreate(tVariant *pVar, SStrToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } +void tVariantCreate(tVariant *pVar, SStrToken *token) { + int32_t ret = 0; + int32_t type = token->type; -void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { memset(pVar, 0, sizeof(tVariant)); - - switch (type) { + + switch (token->type) { case TSDB_DATA_TYPE_BOOL: { - int32_t k = strncasecmp(pz, "true", 4); + int32_t k = strncasecmp(token->z, "true", 4); if (k == 0) { - pVar->i64Key = TSDB_TRUE; + pVar->i64 = TSDB_TRUE; } else { - assert(strncasecmp(pz, "false", 5) == 0); - pVar->i64Key = TSDB_FALSE; + assert(strncasecmp(token->z, "false", 5) == 0); + pVar->i64 = TSDB_FALSE; } + break; } + case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_INT: - pVar->i64Key = strtoll(pz, NULL, 10); + case TSDB_DATA_TYPE_INT:{ + ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true); + if (ret != 0) { + pVar->nType = -1; // -1 means error type + return; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_FLOAT: - pVar->dKey = strtod(pz, NULL); + case TSDB_DATA_TYPE_FLOAT: { + pVar->dKey = strtod(token->z, NULL); break; + } + case TSDB_DATA_TYPE_BINARY: { - pVar->pz = strndup(pz, len); + pVar->pz = strndup(token->z, token->n); pVar->nLen = strdequote(pVar->pz); break; } @@ -74,27 +86,53 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { - pVar->i64Key = GET_INT8_VAL(pz); + pVar->nLen = tDataTypes[type].bytes; + pVar->i64 = GET_INT8_VAL(pz); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + pVar->nLen = tDataTypes[type].bytes; + pVar->u64 = GET_UINT8_VAL(pz); break; } case TSDB_DATA_TYPE_SMALLINT: { - pVar->i64Key = GET_INT16_VAL(pz); + pVar->nLen = tDataTypes[type].bytes; + pVar->i64 = GET_INT16_VAL(pz); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + pVar->nLen = tDataTypes[type].bytes; + pVar->u64 = GET_UINT16_VAL(pz); break; } case TSDB_DATA_TYPE_INT: { - pVar->i64Key = GET_INT32_VAL(pz); + pVar->nLen = tDataTypes[type].bytes; + pVar->i64 = GET_INT32_VAL(pz); + break; + } + case TSDB_DATA_TYPE_UINT: { + pVar->nLen = tDataTypes[type].bytes; + pVar->u64 = GET_UINT32_VAL(pz); break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - pVar->i64Key = GET_INT64_VAL(pz); + pVar->nLen = tDataTypes[type].bytes; + pVar->i64 = GET_INT64_VAL(pz); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + pVar->nLen = tDataTypes[type].bytes; + pVar->u64 = GET_UINT64_VAL(pz); break; } case TSDB_DATA_TYPE_DOUBLE: { + pVar->nLen = tDataTypes[type].bytes; pVar->dKey = GET_DOUBLE_VAL(pz); break; } case TSDB_DATA_TYPE_FLOAT: { + pVar->nLen = tDataTypes[type].bytes; pVar->dKey = GET_FLOAT_VAL(pz); break; } @@ -115,7 +153,8 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 } default: - pVar->i64Key = GET_INT32_VAL(pVar); + pVar->i64 = GET_INT32_VAL(pz); + pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes; } pVar->nType = type; @@ -141,6 +180,11 @@ void tVariantDestroy(tVariant *pVar) { } } +bool tVariantIsValid(tVariant *pVar) { + assert(pVar != NULL); + return isValidDataType(pVar->nType); +} + void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; @@ -159,8 +203,8 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } - if (pSrc->nType >= TSDB_DATA_TYPE_BOOL && pSrc->nType <= TSDB_DATA_TYPE_DOUBLE) { - pDst->i64Key = pSrc->i64Key; + if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { + pDst->i64 = pSrc->i64; } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { // this is only for string array size_t num = taosArrayGetSize(pSrc->arr); pDst->arr = taosArrayInit(num, sizeof(char*)); @@ -172,7 +216,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } if (pDst->nType != TSDB_DATA_TYPE_ARRAY) { - pDst->nLen = tDataTypeDesc[pDst->nType].nSize; + pDst->nLen = tDataTypes[pDst->nType].bytes; } } @@ -189,30 +233,30 @@ int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { return 1; } - switch (p1->nType) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - if (p1->nLen == p2->nLen) { - return memcmp(p1->pz, p2->pz, p1->nLen); - } else { - return p1->nLen > p2->nLen? 1:-1; - } - }; - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - if (p1->dKey == p2->dKey) { - return 0; - } else { - return p1->dKey > p2->dKey? 1:-1; - } - - default: - if (p1->i64Key == p2->i64Key) { - return 0; - } else { - return p1->i64Key > p2->i64Key? 1:-1; - } + if (p1->nType == TSDB_DATA_TYPE_BINARY || p1->nType == TSDB_DATA_TYPE_NCHAR) { + if (p1->nLen == p2->nLen) { + return memcmp(p1->pz, p2->pz, p1->nLen); + } else { + return p1->nLen > p2->nLen? 1:-1; + } + } else if (p1->nType == TSDB_DATA_TYPE_FLOAT || p1->nType == TSDB_DATA_TYPE_DOUBLE) { + if (p1->dKey == p2->dKey) { + return 0; + } else { + return p1->dKey > p2->dKey? 1:-1; + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(p1->nType)) { + if (p1->u64 == p2->u64) { + return 0; + } else { + return p1->u64 > p2->u64? 1:-1; + } + } else { + if (p1->i64 == p2->i64) { + return 0; + } else { + return p1->i64 > p2->i64? 1:-1; + } } } @@ -239,11 +283,15 @@ int32_t tVariantToString(tVariant *pVar, char *dst) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: - return sprintf(dst, "%d", (int32_t)pVar->i64Key); + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + return sprintf(dst, "%d", (int32_t)pVar->i64); case TSDB_DATA_TYPE_BIGINT: - return sprintf(dst, "%" PRId64, pVar->i64Key); - + return sprintf(dst, "%" PRId64, pVar->i64); + case TSDB_DATA_TYPE_UBIGINT: + return sprintf(dst, "%" PRIu64, pVar->u64); case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: return sprintf(dst, "%.9lf", pVar->dKey); @@ -253,121 +301,6 @@ int32_t tVariantToString(tVariant *pVar, char *dst) { } } -#if 0 -static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, bool releaseVariantPtr) { - if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - setNull(pDest, type, tDataTypeDesc[type].nSize); - return 0; - } - - if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - *((int64_t *)pDest) = pVariant->i64Key; - } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { - if ((pVariant->dKey < INT64_MIN) || (pVariant->dKey > INT64_MAX)) { - return -1; - } - - *((int64_t *)pDest) = (int64_t)pVariant->dKey; - } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { - errno = 0; - char *endPtr = NULL; - - SStrToken token = {0}; - token.n = tSQLGetToken(pVariant->pz, &token.type); - - if (token.type == TK_MINUS || token.type == TK_PLUS) { - token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); - } - - if (token.type == TK_FLOAT) { - double v = strtod(pVariant->pz, &endPtr); - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - - if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - return -1; - } - - if ((v < INT64_MIN) || (v > INT64_MAX)) { - return -1; - } - - *((int64_t *)pDest) = (int64_t)v; - } else if (token.type == TK_INTEGER) { - int64_t val = strtoll(pVariant->pz, &endPtr, 10); - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - - if (errno == ERANGE) { - return -1; // data overflow - } - - *((int64_t *)pDest) = val; - } else if (token.type == TK_NULL) { - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - setNull(pDest, type, tDataTypeDesc[type].nSize); - } else { - return -1; - } - - } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { - errno = 0; - wchar_t *endPtr = NULL; - - SStrToken token = {0}; - token.n = tSQLGetToken(pVariant->pz, &token.type); - - if (token.type == TK_MINUS || token.type == TK_PLUS) { - token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); - } - - if (token.type == TK_FLOAT) { - double v = wcstod(pVariant->wpz, &endPtr); - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - - if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - return -1; - } - - if ((v < INT64_MIN) || (v > INT64_MAX)) { - return -1; - } - - *((int64_t *)pDest) = (int64_t)v; - } else if (token.type == TK_NULL) { - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - setNull(pDest, type, tDataTypeDesc[type].nSize); - } else { - int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - - if (errno == ERANGE) { - return -1; // data overflow - } - - *((int64_t *)pDest) = val; - } - } - - return 0; -} -#endif static FORCE_INLINE int32_t convertToBoolImpl(char *pStr, int32_t len) { if ((strncasecmp(pStr, "true", len) == 0) && (len == 4)) { return TSDB_TRUE; @@ -385,15 +318,18 @@ static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) { return TSDB_TRUE; } else if (wcsncasecmp(pstr, L"false", len) == 0 && (len == 5)) { return TSDB_FALSE; + } else if (memcmp(pstr, L"null", wcslen(L"null")) == 0) { + return TSDB_DATA_BOOL_NULL; } else { return -1; } } static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) { - const int32_t INITIAL_ALLOC_SIZE = 20; + const int32_t INITIAL_ALLOC_SIZE = 40; char * pBuf = NULL; - + + // it is a in-place convert type for tVariant, local buffer is needed if (*pDest == pVariant->pz) { pBuf = calloc(1, INITIAL_ALLOC_SIZE); } @@ -413,12 +349,12 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } } else { - if (pVariant->nType >= TSDB_DATA_TYPE_TINYINT && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - sprintf(pBuf == NULL ? *pDest : pBuf, "%" PRId64, pVariant->i64Key); + if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { + sprintf(pBuf == NULL ? *pDest : pBuf, "%" PRId64, pVariant->i64); } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { sprintf(pBuf == NULL ? *pDest : pBuf, "%lf", pVariant->dKey); } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL) { - sprintf(pBuf == NULL ? *pDest : pBuf, "%s", (pVariant->i64Key == TSDB_TRUE) ? "TRUE" : "FALSE"); + sprintf(pBuf == NULL ? *pDest : pBuf, "%s", (pVariant->i64 == TSDB_TRUE) ? "TRUE" : "FALSE"); } else if (pVariant->nType == 0) { // null data setNull(pBuf == NULL ? *pDest : pBuf, TSDB_DATA_TYPE_BINARY, 0); } @@ -432,28 +368,33 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) { return 0; } -// todo handle the error static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { char tmpBuf[40] = {0}; char * pDst = tmpBuf; int32_t nLen = 0; - - if (pVariant->nType >= TSDB_DATA_TYPE_TINYINT && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - nLen = sprintf(pDst, "%" PRId64, pVariant->i64Key); + + // convert the number to string, than convert it to wchar string. + if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { + nLen = sprintf(pDst, "%" PRId64, pVariant->i64); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + nLen = sprintf(pDst, "%"PRIu64, pVariant->u64); } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { nLen = sprintf(pDst, "%lf", pVariant->dKey); } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { pDst = pVariant->pz; nLen = pVariant->nLen; } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL) { - nLen = sprintf(pDst, "%s", (pVariant->i64Key == TSDB_TRUE) ? "TRUE" : "FALSE"); + nLen = sprintf(pDst, "%s", (pVariant->i64 == TSDB_TRUE) ? "TRUE" : "FALSE"); } if (*pDest == pVariant->pz) { wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); - taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); - + bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); + if (!ret) { + return -1; + } + // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { free(pVariant->wpz); @@ -468,11 +409,15 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { pVariant->wpz = (wchar_t *)tmp; } else { - size_t output = -1; - taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); - + int32_t output = 0; + + bool ret = taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); + if (!ret) { + return -1; + } + if (pDestSize != NULL) { - *pDestSize = (int32_t)output; + *pDestSize = output; } } @@ -481,80 +426,59 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { SStrToken stoken = {.z = pStr, .n = len}; - - if (TK_ILLEGAL == isValidNumber(&stoken)) { + if (TK_ILLEGAL == tGetNumericStringType(&stoken)) { return -1; } *value = strtod(pStr, NULL); - return 0; } -static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, int64_t lowBnd, - int64_t upperBnd, bool releaseVariantPtr) { +static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr) { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - setNull((char *)result, type, tDataTypeDesc[type].nSize); + setNull((char *)result, type, tDataTypes[type].bytes); return 0; } - - if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - *result = pVariant->i64Key; - } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { - *result = (int64_t)pVariant->dKey; + + errno = 0; + if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || (pVariant->nType == TSDB_DATA_TYPE_BOOL)) { + *result = pVariant->i64; + } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + *result = pVariant->u64; + } else if (IS_FLOAT_TYPE(pVariant->nType)) { + *result = (int64_t) pVariant->dKey; } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { - errno = 0; - char *endPtr = NULL; - - SStrToken token = {0}; - token.n = tSQLGetToken(pVariant->pz, &token.type); - - if (token.type == TK_MINUS || token.type == TK_PLUS) { - token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); - } - + SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen}; + /*int32_t n = */tSQLGetToken(pVariant->pz, &token.type); + if (token.type == TK_NULL) { if (releaseVariantPtr) { free(pVariant->pz); pVariant->nLen = 0; } - setNull((char *)result, type, tDataTypeDesc[type].nSize); + setNull((char *)result, type, tDataTypes[type].bytes); return 0; } - - SStrToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; - if (TK_ILLEGAL == isValidNumber(&sToken)) { + + // decide if it is a valid number + token.type = tGetNumericStringType(&token); + if (token.type == TK_ILLEGAL) { return -1; } - - if (token.type == TK_FLOAT) { - double v = strtod(pVariant->pz, &endPtr); - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - - if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - return -1; - } - - *result = (int64_t)v; - } else if (token.type == TK_INTEGER) { - int64_t val = strtoll(pVariant->pz, &endPtr, 10); - if (releaseVariantPtr) { - free(pVariant->pz); - pVariant->nLen = 0; - } - - if (errno == ERANGE) { - return -1; // data overflow - } - - *result = val; - } else { + + int64_t res = 0; + int32_t t = tStrToInteger(token.z, token.type, token.n, &res, issigned); + if (t != 0) { return -1; } + + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + + *result = res; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; wchar_t *endPtr = NULL; @@ -583,7 +507,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result free(pVariant->pz); pVariant->nLen = 0; } - setNull((char *)result, type, tDataTypeDesc[type].nSize); + setNull((char *)result, type, tDataTypes[type].bytes); return 0; } else { int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); @@ -599,19 +523,35 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result *result = val; } } - - if ((*result <= lowBnd) || (*result > upperBnd)) { - return -1; + + bool code = false; + switch(type) { + case TSDB_DATA_TYPE_TINYINT: + code = IS_VALID_TINYINT(*result); break; + case TSDB_DATA_TYPE_SMALLINT: + code = IS_VALID_SMALLINT(*result); break; + case TSDB_DATA_TYPE_INT: + code = IS_VALID_INT(*result); break; + case TSDB_DATA_TYPE_BIGINT: + code = IS_VALID_BIGINT(*result); break; + case TSDB_DATA_TYPE_UTINYINT: + code = IS_VALID_UTINYINT(*result); break; + case TSDB_DATA_TYPE_USMALLINT: + code = IS_VALID_USMALLINT(*result); break; + case TSDB_DATA_TYPE_UINT: + code = IS_VALID_UINT(*result); break; + case TSDB_DATA_TYPE_UBIGINT: + code = IS_VALID_UBIGINT(*result); break; } - - return 0; + + return code? 0:-1; } static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) { if (pVariant->nType == TSDB_DATA_TYPE_BOOL) { - *pDest = pVariant->i64Key; // in order to be compatible to null of bool - } else if (pVariant->nType >= TSDB_DATA_TYPE_TINYINT && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - *pDest = ((pVariant->i64Key != 0) ? TSDB_TRUE : TSDB_FALSE); + *pDest = pVariant->i64; // in order to be compatible to null of bool + } else if (IS_NUMERIC_TYPE(pVariant->nType)) { + *pDest = ((pVariant->i64 != 0) ? TSDB_TRUE : TSDB_FALSE); } else if (pVariant->nType == TSDB_DATA_TYPE_FLOAT || pVariant->nType == TSDB_DATA_TYPE_DOUBLE) { *pDest = ((pVariant->dKey != 0) ? TSDB_TRUE : TSDB_FALSE); } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { @@ -638,8 +578,6 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) { /* * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType * to column type defined in schema - * - * todo handle the return value */ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) { if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) { @@ -647,57 +585,82 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } errno = 0; // reset global error code - + int64_t result; + switch (type) { case TSDB_DATA_TYPE_BOOL: { - int64_t dst = 0; - if (convertToBool(pVariant, &dst) < 0) { + if (convertToBool(pVariant, &result) < 0) { return -1; } - *(int8_t *)payload = (int8_t)dst; + + *(int8_t *)payload = (int8_t)result; break; } case TSDB_DATA_TYPE_TINYINT: { - int64_t result = 0; - if (convertToInteger(pVariant, &result, type, INT8_MIN, INT8_MAX, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false) < 0) { return -1; } - - *((int8_t *)payload) = (int8_t)result; + *((int8_t *)payload) = (int8_t) result; + break; + } + + case TSDB_DATA_TYPE_UTINYINT: { + if (convertToInteger(pVariant, &result, type, false, false) < 0) { + return -1; + } + *((uint8_t *)payload) = (uint8_t) result; break; } case TSDB_DATA_TYPE_SMALLINT: { - int64_t result = 0; - if (convertToInteger(pVariant, &result, type, INT16_MIN, INT16_MAX, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false) < 0) { return -1; } - *((int16_t *)payload) = (int16_t)result; break; } + + case TSDB_DATA_TYPE_USMALLINT: { + if (convertToInteger(pVariant, &result, type, false, false) < 0) { + return -1; + } + *((uint16_t *)payload) = (uint16_t)result; + break; + } case TSDB_DATA_TYPE_INT: { - int64_t result = 0; - if (convertToInteger(pVariant, &result, type, INT32_MIN, INT32_MAX, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false) < 0) { return -1; } - *((int32_t *)payload) = (int32_t)result; break; } + + case TSDB_DATA_TYPE_UINT: { + if (convertToInteger(pVariant, &result, type, false, false) < 0) { + return -1; + } + *((uint32_t *)payload) = (uint32_t)result; + break; + } case TSDB_DATA_TYPE_BIGINT: { - int64_t result = 0; - if (convertToInteger(pVariant, &result, type, INT64_MIN, INT64_MAX, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false) < 0) { return -1; } - *((int64_t *)payload) = (int64_t)result; break; } - + + case TSDB_DATA_TYPE_UBIGINT: { + if (convertToInteger(pVariant, &result, type, false, false) < 0) { + return -1; + } + *((uint64_t *)payload) = (uint64_t)result; + break; + } + case TSDB_DATA_TYPE_FLOAT: { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { if (strncasecmp(TSDB_DATA_NULL_STR_L, pVariant->pz, pVariant->nLen) == 0 && @@ -705,21 +668,20 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; return 0; } else { - double value; - int32_t ret; - ret = convertToDouble(pVariant->pz, pVariant->nLen, &value); + double value = -1; + int32_t ret = convertToDouble(pVariant->pz, pVariant->nLen, &value); if ((errno == ERANGE && (float)value == -1) || (ret != 0)) { return -1; } SET_FLOAT_VAL(payload, value); } - } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - SET_FLOAT_VAL(payload, pVariant->i64Key); - } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { + } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + SET_FLOAT_VAL(payload, pVariant->i64); + } else if (IS_FLOAT_TYPE(pVariant->nType)) { SET_FLOAT_VAL(payload, pVariant->dKey); } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; + *((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL; return 0; } @@ -745,9 +707,9 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu SET_DOUBLE_VAL(payload, value); } - } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - SET_DOUBLE_VAL(payload, pVariant->i64Key); - } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { + } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + SET_DOUBLE_VAL(payload, pVariant->i64); + } else if (IS_FLOAT_TYPE(pVariant->nType)) { SET_DOUBLE_VAL(payload, pVariant->dKey); } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; @@ -755,9 +717,10 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } double dv = GET_DOUBLE_VAL(payload); - if (isinf(dv) || isnan(dv) || dv > DBL_MAX || dv < -DBL_MAX) { + if (errno == ERANGE || isinf(dv) || isnan(dv)) { return -1; } + break; } @@ -794,7 +757,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu if (pVariant->nType == TSDB_DATA_TYPE_NULL) { *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; } else { - *((int64_t *)payload) = pVariant->i64Key; + *((int64_t *)payload) = pVariant->i64; } break; } @@ -805,7 +768,9 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL; } else { if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { - toNchar(pVariant, &payload, &newlen); + if (toNchar(pVariant, &payload, &newlen) != 0) { + return -1; + } } else { wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen); } @@ -817,9 +782,11 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu char *p = varDataVal(payload); if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { - toNchar(pVariant, &p, &newlen); + if (toNchar(pVariant, &p, &newlen) != 0) { + return -1; + } } else { - wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen); + memcpy(p, pVariant->wpz, pVariant->nLen); newlen = pVariant->nLen; } @@ -848,7 +815,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { switch (type) { case TSDB_DATA_TYPE_BOOL: { // bool - if (convertToBool(pVariant, &pVariant->i64Key) < 0) { + if (convertToBool(pVariant, &pVariant->i64) < 0) { return -1; } @@ -859,7 +826,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: { - convertToInteger(pVariant, &(pVariant->i64Key), type, INT64_MIN, INT64_MAX, true); + convertToInteger(pVariant, &(pVariant->i64), type, true, true); pVariant->nType = TSDB_DATA_TYPE_BIGINT; break; } @@ -886,7 +853,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { free(pVariant->pz); pVariant->dKey = v; } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - pVariant->dKey = (double)(pVariant->i64Key); + pVariant->dKey = (double)(pVariant->i64); } pVariant->nType = TSDB_DATA_TYPE_DOUBLE; @@ -901,7 +868,9 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { } case TSDB_DATA_TYPE_NCHAR: { if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { - toNchar(pVariant, &pVariant->pz, &pVariant->nLen); + if (toNchar(pVariant, &pVariant->pz, &pVariant->nLen) != 0) { + return -1; + } } pVariant->nType = type; break; @@ -910,5 +879,3 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { return 0; } - - diff --git a/src/connector/C#/TDengineDriver.cs b/src/connector/C#/TDengineDriver.cs index b6f143e1813d60c1ac4ae8356efdca4929c51345..205269501d376a4753b3aedbfa8d512b2df31600 100644 --- a/src/connector/C#/TDengineDriver.cs +++ b/src/connector/C#/TDengineDriver.cs @@ -19,136 +19,137 @@ using System.Runtime.InteropServices; namespace TDengineDriver { - enum TDengineDataType { - TSDB_DATA_TYPE_NULL = 0, // 1 bytes - TSDB_DATA_TYPE_BOOL = 1, // 1 bytes - TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes - TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes - TSDB_DATA_TYPE_INT = 4, // 4 bytes - TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes - TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes - TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes - TSDB_DATA_TYPE_BINARY = 8, // string - TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes - TSDB_DATA_TYPE_NCHAR = 10 // unicode string - } - - enum TDengineInitOption - { - TSDB_OPTION_LOCALE = 0, - TSDB_OPTION_CHARSET = 1, - TSDB_OPTION_TIMEZONE = 2, - TDDB_OPTION_CONFIGDIR = 3, - TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 - } - - class TDengineMeta - { - public string name; - public short size; - public byte type; - public string TypeName() + enum TDengineDataType { - switch ((TDengineDataType)type) - { - case TDengineDataType.TSDB_DATA_TYPE_BOOL: - return "BOOLEAN"; - case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - return "BYTE"; - case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: - return "SHORT"; - case TDengineDataType.TSDB_DATA_TYPE_INT: - return "INT"; - case TDengineDataType.TSDB_DATA_TYPE_BIGINT: - return "LONG"; - case TDengineDataType.TSDB_DATA_TYPE_FLOAT: - return "FLOAT"; - case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: - return "DOUBLE"; - case TDengineDataType.TSDB_DATA_TYPE_BINARY: - return "STRING"; - case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: - return "TIMESTAMP"; - case TDengineDataType.TSDB_DATA_TYPE_NCHAR: - return "NCHAR"; - default: - return "undefine"; - } + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10 // unicode string } - } - class TDengine - { - public const int TSDB_CODE_SUCCESS = 0; + enum TDengineInitOption + { + TSDB_OPTION_LOCALE = 0, + TSDB_OPTION_CHARSET = 1, + TSDB_OPTION_TIMEZONE = 2, + TDDB_OPTION_CONFIGDIR = 3, + TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 + } - [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] - static extern public void Init(); + class TDengineMeta + { + public string name; + public short size; + public byte type; + public string TypeName() + { + switch ((TDengineDataType)type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + return "BOOLEAN"; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + return "BYTE"; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + return "SHORT"; + case TDengineDataType.TSDB_DATA_TYPE_INT: + return "INT"; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + return "LONG"; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + return "STRING"; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + return "NCHAR"; + default: + return "undefine"; + } + } + } - [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] - static extern public void Cleanup(); + class TDengine + { + public const int TSDB_CODE_SUCCESS = 0; - [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] - static extern public void Options(int option, string value); + [DllImport("taos", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] + static extern public void Init(); - [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr Connect(string ip, string user, string password, string db, short port); + [DllImport("taos", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); - [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] - static extern private IntPtr taos_errstr(IntPtr res); - static public string Error(IntPtr res) - { - IntPtr errPtr = taos_errstr(res); - return Marshal.PtrToStringAnsi(errPtr); - } + [DllImport("taos", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] + static extern public void Options(int option, string value); - [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] - static extern public int ErrorNo(IntPtr res); + [DllImport("taos", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); - [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr Query(IntPtr conn, string sqlstr); + [DllImport("taos", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) + { + IntPtr errPtr = taos_errstr(res); + return Marshal.PtrToStringAnsi(errPtr); + } - [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] - static extern public int AffectRows(IntPtr res); + [DllImport("taos", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] - static extern public int FieldCount(IntPtr res); + [DllImport("taos", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); - [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] - static extern private IntPtr taos_fetch_fields(IntPtr res); - static public List FetchFields(IntPtr res) - { - const int fieldSize = 68; - - List metas = new List(); - if (res == IntPtr.Zero) - { - return metas; - } - - int fieldCount = FieldCount(res); - IntPtr fieldsPtr = taos_fetch_fields(res); - - for (int i = 0; i < fieldCount; ++i) - { - int offset = i * fieldSize; - - TDengineMeta meta = new TDengineMeta(); - meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); - meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); - meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); - metas.Add(meta); - } - - return metas; - } + [DllImport("taos", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr FetchRows(IntPtr res); + [DllImport("taos", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] - static extern public IntPtr FreeResult(IntPtr res); + [DllImport("taos", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) + { + const int fieldSize = 68; - [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] - static extern public int Close(IntPtr taos); - } -} \ No newline at end of file + List metas = new List(); + if (res == IntPtr.Zero) + { + return metas; + } + + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); + + for (int i = 0; i < fieldCount; ++i) + { + int offset = i * fieldSize; + + TDengineMeta meta = new TDengineMeta(); + meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); + metas.Add(meta); + } + + return metas; + } + + [DllImport("taos", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); + } +} diff --git a/src/connector/go b/src/connector/go index 8c58c512b6acda8bcdfa48fdc7140227b5221766..050667e5b4d0eafa5387e4283e713559b421203f 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 +Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index d598db167eb256fe67409b7bb3d0eb7fffc3ff8c..32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c +Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath deleted file mode 100644 index a5d95095ccaaf9549ee22ff2fbf684bfa43d31c9..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/.classpath +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/connector/jdbc/.project b/src/connector/jdbc/.project deleted file mode 100644 index 656ab58d205bf0e01c151728a75e5ab543facee6..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - taos-jdbcdriver - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - - diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 0eb3eb21cec7d79fc5fec5f7b0400a0a5a39a137..3c50ac566b5fe49a619a50c9eac446285a2b431c 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.21-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 4564bde81e450f9c44be52ef69918dc0b3ada26f..1c24b621efe52e152ac8ea70ef3c2afdbc72d5b0 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.14 + 2.0.21 jar JDBCDriver @@ -36,7 +36,6 @@ - commons-logging diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 7e087ebd9b05709518d45b8096cbd2b051c4a214..7a421eff2296cc8b61ec6e07174747ef8e224ff0 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.14 + 2.0.21 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -34,21 +34,9 @@ UTF-8 1.8 3.6.0 - 1.1.2 - 3.5 + - - commons-logging - commons-logging - ${commons-logging.version} - - - * - * - - - junit junit @@ -62,11 +50,6 @@ httpclient 4.5.8 - - org.apache.commons - commons-lang3 - 3.9 - com.alibaba fastjson @@ -74,6 +57,7 @@ + @@ -112,12 +96,18 @@ maven-surefire-plugin 2.12.4 + pertest + ${maven.test.jvmargs} **/*Test.java - **/BatchInsertTest.java + **/AppMemoryLeakTest.java + **/AuthenticationTest.java + **/TaosInfoMonitorTest.java **/FailOverTest.java + **/InvalidResultSetPointerTest.java + **/RestfulConnectionTest.java true diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java new file mode 100644 index 0000000000000000000000000000000000000000..a2496b0099fe27c6380b1e2bac954ac361c939ba --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java @@ -0,0 +1,530 @@ +package com.taosdata.jdbc; + +import java.sql.*; +import java.util.Enumeration; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.*; + +public abstract class AbstractConnection extends WrapperImpl implements Connection { + + protected volatile boolean isClosed; + protected volatile String catalog; + protected volatile Properties clientInfoProps = new Properties(); + + @Override + public abstract Statement createStatement() throws SQLException; + + @Override + public abstract PreparedStatement prepareStatement(String sql) throws SQLException; + + @Override + public CallableStatement prepareCall(String sql) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public String nativeSQL(String sql) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + // do nothing + return sql; + } + + @Override + public void setAutoCommit(boolean autoCommit) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + //do nothing + } + + @Override + public boolean getAutoCommit() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return true; + } + + @Override + public void commit() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public void rollback() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public abstract void close() throws SQLException; + + @Override + public abstract boolean isClosed() throws SQLException; + + @Override + public abstract DatabaseMetaData getMetaData() throws SQLException; + + @Override + public void setReadOnly(boolean readOnly) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + //do nothing + } + + @Override + public boolean isReadOnly() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return true; + } + + @Override + public void setCatalog(String catalog) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + /* + try (Statement stmt = createStatement()) { + boolean execute = stmt.execute("use " + catalog); + if (execute) + this.catalog = catalog; + } catch (SQLException e) { + // do nothing + } + */ + + this.catalog = catalog; + } + + @Override + public String getCatalog() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return this.catalog; + } + + @Override + public void setTransactionIsolation(int level) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (level) { + case Connection.TRANSACTION_NONE: + break; + case Connection.TRANSACTION_READ_UNCOMMITTED: + case Connection.TRANSACTION_READ_COMMITTED: + case Connection.TRANSACTION_REPEATABLE_READ: + case Connection.TRANSACTION_SERIALIZABLE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + //do nothing + } + + @Override + public int getTransactionIsolation() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return Connection.TRANSACTION_NONE; + } + + @Override + public SQLWarning getWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return null; + } + + @Override + public void clearWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetType) { + case ResultSet.TYPE_FORWARD_ONLY: + break; + case ResultSet.TYPE_SCROLL_INSENSITIVE: + case ResultSet.TYPE_SCROLL_SENSITIVE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + switch (resultSetConcurrency) { + case ResultSet.CONCUR_READ_ONLY: + break; + case ResultSet.CONCUR_UPDATABLE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + return createStatement(); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetType) { + case ResultSet.TYPE_FORWARD_ONLY: + break; + case ResultSet.TYPE_SCROLL_INSENSITIVE: + case ResultSet.TYPE_SCROLL_SENSITIVE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + switch (resultSetConcurrency) { + case ResultSet.CONCUR_READ_ONLY: + break; + case ResultSet.CONCUR_UPDATABLE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + return prepareStatement(sql); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Map> getTypeMap() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTypeMap(Map> map) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setHoldability(int holdability) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (holdability) { + case ResultSet.HOLD_CURSORS_OVER_COMMIT: + break; + case ResultSet.CLOSE_CURSORS_AT_COMMIT: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + //do nothing + } + + @Override + public int getHoldability() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public Savepoint setSavepoint() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Savepoint setSavepoint(String name) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void rollback(Savepoint savepoint) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetHoldability) { + case ResultSet.HOLD_CURSORS_OVER_COMMIT: + break; + case ResultSet.CLOSE_CURSORS_AT_COMMIT: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + return createStatement(resultSetType, resultSetConcurrency); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetHoldability) { + case ResultSet.HOLD_CURSORS_OVER_COMMIT: + break; + case ResultSet.CLOSE_CURSORS_AT_COMMIT: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + return prepareStatement(sql, resultSetType, resultSetConcurrency); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (autoGeneratedKeys) { + case Statement.RETURN_GENERATED_KEYS: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + case Statement.NO_GENERATED_KEYS: + break; + } + return prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Clob createClob() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Blob createBlob() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public NClob createNClob() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLXML createSQLXML() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isValid(int timeout) throws SQLException { + //true if the connection is valid, false otherwise + if (isClosed()) + return false; + if (timeout < 0) //SQLException - if the value supplied for timeout is less then 0 + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + + ExecutorService executor = Executors.newCachedThreadPool(); + Future future = executor.submit(() -> { + int status; + try (Statement stmt = createStatement()) { + ResultSet resultSet = stmt.executeQuery("select server_status()"); + resultSet.next(); + status = resultSet.getInt("server_status()"); + resultSet.close(); + } + return status == 1 ? true : false; + }); + + boolean status = false; + try { + if (timeout == 0) + status = future.get(); + else + status = future.get(timeout, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } catch (TimeoutException e) { + future.cancel(true); + status = false; + } finally { + executor.shutdownNow(); + } + return status; + } + + @Override + public void setClientInfo(String name, String value) throws SQLClientInfoException { + if (isClosed) + throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); + + if (clientInfoProps == null) + clientInfoProps = new Properties(); + clientInfoProps.setProperty(name, value); + } + + @Override + public void setClientInfo(Properties properties) throws SQLClientInfoException { + if (isClosed) + throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); + + + for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { + String name = (String) enumer.nextElement(); + clientInfoProps.put(name, properties.getProperty(name)); + } + } + + @Override + public String getClientInfo(String name) throws SQLException { + if (isClosed) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return clientInfoProps.getProperty(name); + } + + @Override + public Properties getClientInfo() throws SQLException { + if (isClosed) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return clientInfoProps; + } + + @Override + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setSchema(String schema) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + //do nothing + } + + @Override + public String getSchema() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return null; + } + + @Override + public void abort(Executor executor) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + if (milliseconds < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getNetworkTimeout() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java new file mode 100644 index 0000000000000000000000000000000000000000..5dcaa77ebd4a15087785a6a9b642b85f160f5287 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java @@ -0,0 +1,1256 @@ +package com.taosdata.jdbc; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractDatabaseMetaData extends WrapperImpl implements DatabaseMetaData { + + private final static String PRODUCT_NAME = "TDengine"; + private final static String PRODUCT_VESION = "2.0.x.x"; + private final static String DRIVER_VERSION = "2.0.x"; + private final static int DRIVER_MAJAR_VERSION = 2; + private final static int DRIVER_MINOR_VERSION = 0; + + public boolean allProceduresAreCallable() throws SQLException { + return false; + } + + public boolean allTablesAreSelectable() throws SQLException { + return false; + } + + public abstract String getURL() throws SQLException; + + public abstract String getUserName() throws SQLException; + + public boolean isReadOnly() throws SQLException { + return false; + } + + public boolean nullsAreSortedHigh() throws SQLException { + return false; + } + + public boolean nullsAreSortedLow() throws SQLException { + return !nullsAreSortedHigh(); + } + + public boolean nullsAreSortedAtStart() throws SQLException { + return true; + } + + public boolean nullsAreSortedAtEnd() throws SQLException { + return !nullsAreSortedAtStart(); + } + + public String getDatabaseProductName() throws SQLException { + return PRODUCT_NAME; + } + + public String getDatabaseProductVersion() throws SQLException { + return PRODUCT_VESION; + } + + public abstract String getDriverName() throws SQLException; + + public String getDriverVersion() throws SQLException { + return DRIVER_VERSION; + } + + public int getDriverMajorVersion() { + return DRIVER_MAJAR_VERSION; + } + + public int getDriverMinorVersion() { + return DRIVER_MINOR_VERSION; + } + + public boolean usesLocalFiles() throws SQLException { + return false; + } + + public boolean usesLocalFilePerTable() throws SQLException { + return false; + } + + public boolean supportsMixedCaseIdentifiers() throws SQLException { + //像database、table这些对象的标识符，在存储时是否采用大小写混合的模式 + return false; + } + + public boolean storesUpperCaseIdentifiers() throws SQLException { + return false; + } + + public boolean storesLowerCaseIdentifiers() throws SQLException { + return true; + } + + public boolean storesMixedCaseIdentifiers() throws SQLException { + return false; + } + + public boolean supportsMixedCaseQuotedIdentifiers() throws SQLException { + return false; + } + + public boolean storesUpperCaseQuotedIdentifiers() throws SQLException { + return false; + } + + public boolean storesLowerCaseQuotedIdentifiers() throws SQLException { + return false; + } + + public boolean storesMixedCaseQuotedIdentifiers() throws SQLException { + return false; + } + + public String getIdentifierQuoteString() throws SQLException { + return " "; + } + + public String getSQLKeywords() throws SQLException { + return null; + } + + public String getNumericFunctions() throws SQLException { + return null; + } + + public String getStringFunctions() throws SQLException { + return null; + } + + public String getSystemFunctions() throws SQLException { + return null; + } + + public String getTimeDateFunctions() throws SQLException { + return null; + } + + public String getSearchStringEscape() throws SQLException { + return null; + } + + public String getExtraNameCharacters() throws SQLException { + return null; + } + + public boolean supportsAlterTableWithAddColumn() throws SQLException { + return true; + } + + public boolean supportsAlterTableWithDropColumn() throws SQLException { + return true; + } + + public boolean supportsColumnAliasing() throws SQLException { + return true; + } + + public boolean nullPlusNonNullIsNull() throws SQLException { + // null + non-null != null + return false; + } + + public boolean supportsConvert() throws SQLException { + // 是否支持转换函数convert + return false; + } + + public boolean supportsConvert(int fromType, int toType) throws SQLException { + return false; + } + + public boolean supportsTableCorrelationNames() throws SQLException { + return false; + } + + public boolean supportsDifferentTableCorrelationNames() throws SQLException { + return false; + } + + public boolean supportsExpressionsInOrderBy() throws SQLException { + return false; + } + + public boolean supportsOrderByUnrelated() throws SQLException { + return false; + } + + public boolean supportsGroupBy() throws SQLException { + return true; + } + + public boolean supportsGroupByUnrelated() throws SQLException { + return false; + } + + public boolean supportsGroupByBeyondSelect() throws SQLException { + return false; + } + + public boolean supportsLikeEscapeClause() throws SQLException { + return false; + } + + public boolean supportsMultipleResultSets() throws SQLException { + return false; + } + + public boolean supportsMultipleTransactions() throws SQLException { + return false; + } + + public boolean supportsNonNullableColumns() throws SQLException { + return false; + } + + public boolean supportsMinimumSQLGrammar() throws SQLException { + return false; + } + + public boolean supportsCoreSQLGrammar() throws SQLException { + return false; + } + + public boolean supportsExtendedSQLGrammar() throws SQLException { + return false; + } + + public boolean supportsANSI92EntryLevelSQL() throws SQLException { + return false; + } + + public boolean supportsANSI92IntermediateSQL() throws SQLException { + return false; + } + + public boolean supportsANSI92FullSQL() throws SQLException { + return false; + } + + public boolean supportsIntegrityEnhancementFacility() throws SQLException { + return false; + } + + public boolean supportsOuterJoins() throws SQLException { + return false; + } + + public boolean supportsFullOuterJoins() throws SQLException { + return false; + } + + public boolean supportsLimitedOuterJoins() throws SQLException { + return false; + } + + public String getSchemaTerm() throws SQLException { + return null; + } + + public String getProcedureTerm() throws SQLException { + return null; + } + + public String getCatalogTerm() throws SQLException { + return "database"; + } + + public boolean isCatalogAtStart() throws SQLException { + return true; + } + + public String getCatalogSeparator() throws SQLException { + return "."; + } + + public boolean supportsSchemasInDataManipulation() throws SQLException { + return false; + } + + public boolean supportsSchemasInProcedureCalls() throws SQLException { + return false; + } + + public boolean supportsSchemasInTableDefinitions() throws SQLException { + return false; + } + + public boolean supportsSchemasInIndexDefinitions() throws SQLException { + return false; + } + + public boolean supportsSchemasInPrivilegeDefinitions() throws SQLException { + return false; + } + + public boolean supportsCatalogsInDataManipulation() throws SQLException { + return true; + } + + public boolean supportsCatalogsInProcedureCalls() throws SQLException { + return false; + } + + public boolean supportsCatalogsInTableDefinitions() throws SQLException { + return false; + } + + public boolean supportsCatalogsInIndexDefinitions() throws SQLException { + return false; + } + + public boolean supportsCatalogsInPrivilegeDefinitions() throws SQLException { + return false; + } + + public boolean supportsPositionedDelete() throws SQLException { + return false; + } + + public boolean supportsPositionedUpdate() throws SQLException { + return false; + } + + public boolean supportsSelectForUpdate() throws SQLException { + return false; + } + + public boolean supportsStoredProcedures() throws SQLException { + return false; + } + + public boolean supportsSubqueriesInComparisons() throws SQLException { + return false; + } + + public boolean supportsSubqueriesInExists() throws SQLException { + return false; + } + + public boolean supportsSubqueriesInIns() throws SQLException { + return false; + } + + public boolean supportsSubqueriesInQuantifieds() throws SQLException { + return false; + } + + public boolean supportsCorrelatedSubqueries() throws SQLException { + return false; + } + + public boolean supportsUnion() throws SQLException { + return false; + } + + public boolean supportsUnionAll() throws SQLException { + return false; + } + + public boolean supportsOpenCursorsAcrossCommit() throws SQLException { + return false; + } + + public boolean supportsOpenCursorsAcrossRollback() throws SQLException { + return false; + } + + public boolean supportsOpenStatementsAcrossCommit() throws SQLException { + return false; + } + + public boolean supportsOpenStatementsAcrossRollback() throws SQLException { + return false; + } + + public int getMaxBinaryLiteralLength() throws SQLException { + return 0; + } + + public int getMaxCharLiteralLength() throws SQLException { + return 0; + } + + public int getMaxColumnNameLength() throws SQLException { + return 0; + } + + public int getMaxColumnsInGroupBy() throws SQLException { + return 0; + } + + public int getMaxColumnsInIndex() throws SQLException { + return 0; + } + + public int getMaxColumnsInOrderBy() throws SQLException { + return 0; + } + + public int getMaxColumnsInSelect() throws SQLException { + return 0; + } + + public int getMaxColumnsInTable() throws SQLException { + return 0; + } + + public int getMaxConnections() throws SQLException { + return 0; + } + + public int getMaxCursorNameLength() throws SQLException { + return 0; + } + + public int getMaxIndexLength() throws SQLException { + return 0; + } + + public int getMaxSchemaNameLength() throws SQLException { + return 0; + } + + public int getMaxProcedureNameLength() throws SQLException { + return 0; + } + + public int getMaxCatalogNameLength() throws SQLException { + return 0; + } + + public int getMaxRowSize() throws SQLException { + return 0; + } + + public boolean doesMaxRowSizeIncludeBlobs() throws SQLException { + return false; + } + + public int getMaxStatementLength() throws SQLException { + return 0; + } + + public int getMaxStatements() throws SQLException { + return 0; + } + + public int getMaxTableNameLength() throws SQLException { + return 0; + } + + public int getMaxTablesInSelect() throws SQLException { + return 0; + } + + public int getMaxUserNameLength() throws SQLException { + return 0; + } + + public int getDefaultTransactionIsolation() throws SQLException { + return Connection.TRANSACTION_NONE; + } + + public boolean supportsTransactions() throws SQLException { + return false; + } + + public boolean supportsTransactionIsolationLevel(int level) throws SQLException { + if (level == Connection.TRANSACTION_NONE) + return true; + return false; + } + + public boolean supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { + return false; + } + + public boolean supportsDataManipulationTransactionsOnly() throws SQLException { + return false; + } + + public boolean dataDefinitionCausesTransactionCommit() throws SQLException { + return false; + } + + public boolean dataDefinitionIgnoredInTransactions() throws SQLException { + return false; + } + + public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) throws SQLException { + return null; + } + + public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, String columnNamePattern) throws SQLException { + return null; + } + + public abstract ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException; + + protected ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types, Connection connection) throws SQLException { + try (Statement stmt = connection.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + List columnMetaDataList = new ArrayList<>(); + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(1); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(2); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(3); + col3.setColName("TABLE_NAME"); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(4); + col4.setColName("TABLE_TYPE"); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + ColumnMetaData col5 = new ColumnMetaData(); + col5.setColIndex(5); + col5.setColName("REMARKS"); + col5.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col5); + ColumnMetaData col6 = new ColumnMetaData(); + col6.setColIndex(6); + col6.setColName("TYPE_CAT"); + col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col6); + ColumnMetaData col7 = new ColumnMetaData(); + col7.setColIndex(7); + col7.setColName("TYPE_SCHEM"); + col7.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col7); + ColumnMetaData col8 = new ColumnMetaData(); + col8.setColIndex(8); + col8.setColName("TYPE_NAME"); + col8.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col8); + ColumnMetaData col9 = new ColumnMetaData(); + col9.setColIndex(9); + col9.setColName("SELF_REFERENCING_COL_NAME"); + col9.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col9); + ColumnMetaData col10 = new ColumnMetaData(); + col10.setColIndex(10); + col10.setColName("REF_GENERATION"); + col10.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col10); + resultSet.setColumnMetaDataList(columnMetaDataList); + + List rowDataList = new ArrayList<>(); + ResultSet tables = stmt.executeQuery("show tables"); + while (tables.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(10); + rowData.setString(0, dbname); //table_cat + rowData.setString(1, null); //TABLE_SCHEM + rowData.setString(2, tables.getString("table_name")); //TABLE_NAME + rowData.setString(3, "TABLE"); //TABLE_TYPE + rowData.setString(4, ""); //REMARKS + rowDataList.add(rowData); + } + + ResultSet stables = stmt.executeQuery("show stables"); + while (stables.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(10); + rowData.setString(0, dbname); //TABLE_CAT + rowData.setString(1, null); //TABLE_SCHEM + rowData.setString(2, stables.getString("name")); //TABLE_NAME + rowData.setString(3, "TABLE"); //TABLE_TYPE + rowData.setString(4, "STABLE"); //REMARKS + rowDataList.add(rowData); + } + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } + + public ResultSet getSchemas() throws SQLException { + return getEmptyResultSet(); + } + + public abstract ResultSet getCatalogs() throws SQLException; + + public ResultSet getTableTypes() throws SQLException { + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + List columnMetaDataList = new ArrayList<>(); + ColumnMetaData colMetaData = new ColumnMetaData(); + colMetaData.setColIndex(0); + colMetaData.setColName("TABLE_TYPE"); + colMetaData.setColSize(10); + colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(colMetaData); + resultSet.setColumnMetaDataList(columnMetaDataList); + + // set up rowDataList + List rowDataList = new ArrayList<>(); + TSDBResultSetRowData rowData = new TSDBResultSetRowData(1); + rowData.setString(0, "TABLE"); + rowDataList.add(rowData); + rowData = new TSDBResultSetRowData(1); + rowData.setString(0, "STABLE"); + rowDataList.add(rowData); + resultSet.setRowDataList(rowDataList); + + return resultSet; + } + + public abstract ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException; + + protected ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern, Connection conn) { + try (Statement stmt = conn.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(1); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + // TABLE_SCHEM + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(2); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + // TABLE_NAME + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(3); + col3.setColName("TABLE_NAME"); + col3.setColSize(193); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + // COLUMN_NAME + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(4); + col4.setColName("COLUMN_NAME"); + col4.setColSize(65); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + // DATA_TYPE + ColumnMetaData col5 = new ColumnMetaData(); + col5.setColIndex(5); + col5.setColName("DATA_TYPE"); + col5.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col5); + // TYPE_NAME + ColumnMetaData col6 = new ColumnMetaData(); + col6.setColIndex(6); + col6.setColName("TYPE_NAME"); + col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col6); + // COLUMN_SIZE + ColumnMetaData col7 = new ColumnMetaData(); + col7.setColIndex(7); + col7.setColName("COLUMN_SIZE"); + col7.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col7); + // BUFFER_LENGTH, not used + ColumnMetaData col8 = new ColumnMetaData(); + col8.setColIndex(8); + col8.setColName("BUFFER_LENGTH"); + columnMetaDataList.add(col8); + // DECIMAL_DIGITS + ColumnMetaData col9 = new ColumnMetaData(); + col9.setColIndex(9); + col9.setColName("DECIMAL_DIGITS"); + col9.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col9); + // add NUM_PREC_RADIX + ColumnMetaData col10 = new ColumnMetaData(); + col10.setColIndex(10); + col10.setColName("NUM_PREC_RADIX"); + col10.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col10); + // NULLABLE + ColumnMetaData col11 = new ColumnMetaData(); + col11.setColIndex(11); + col11.setColName("NULLABLE"); + col11.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col11); + // REMARKS + ColumnMetaData col12 = new ColumnMetaData(); + col12.setColIndex(12); + col12.setColName("REMARKS"); + col12.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col12); + // COLUMN_DEF + ColumnMetaData col13 = new ColumnMetaData(); + col13.setColIndex(13); + col13.setColName("COLUMN_DEF"); + col13.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col13); + //SQL_DATA_TYPE + ColumnMetaData col14 = new ColumnMetaData(); + col14.setColIndex(14); + col14.setColName("SQL_DATA_TYPE"); + col14.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col14); + //SQL_DATETIME_SUB + ColumnMetaData col15 = new ColumnMetaData(); + col15.setColIndex(15); + col15.setColName("SQL_DATETIME_SUB"); + col15.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col15); + //CHAR_OCTET_LENGTH + ColumnMetaData col16 = new ColumnMetaData(); + col16.setColIndex(16); + col16.setColName("CHAR_OCTET_LENGTH"); + col16.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col16); + //ORDINAL_POSITION + ColumnMetaData col17 = new ColumnMetaData(); + col17.setColIndex(17); + col17.setColName("ORDINAL_POSITION"); + col17.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col17); + // IS_NULLABLE + ColumnMetaData col18 = new ColumnMetaData(); + col18.setColIndex(18); + col18.setColName("IS_NULLABLE"); + col18.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col18); + //SCOPE_CATALOG + ColumnMetaData col19 = new ColumnMetaData(); + col19.setColIndex(19); + col19.setColName("SCOPE_CATALOG"); + col19.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col19); + //SCOPE_SCHEMA + ColumnMetaData col20 = new ColumnMetaData(); + col20.setColIndex(20); + col20.setColName("SCOPE_SCHEMA"); + col20.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col20); + //SCOPE_TABLE + ColumnMetaData col21 = new ColumnMetaData(); + col21.setColIndex(21); + col21.setColName("SCOPE_TABLE"); + col21.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col21); + //SOURCE_DATA_TYPE + ColumnMetaData col22 = new ColumnMetaData(); + col22.setColIndex(22); + col22.setColName("SOURCE_DATA_TYPE"); + col22.setColType(TSDBConstants.TSDB_DATA_TYPE_SMALLINT); + columnMetaDataList.add(col22); + //IS_AUTOINCREMENT + ColumnMetaData col23 = new ColumnMetaData(); + col23.setColIndex(23); + col23.setColName("IS_AUTOINCREMENT"); + col23.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col23); + //IS_GENERATEDCOLUMN + ColumnMetaData col24 = new ColumnMetaData(); + col24.setColIndex(24); + col24.setColName("IS_GENERATEDCOLUMN"); + col24.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col24); + + resultSet.setColumnMetaDataList(columnMetaDataList); + // set up rowDataList + ResultSet rs = stmt.executeQuery("describe " + dbname + "." + tableNamePattern); + List rowDataList = new ArrayList<>(); + int index = 0; + while (rs.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(24); + // set TABLE_CAT + rowData.setString(0, dbname); + // set TABLE_NAME + rowData.setString(2, tableNamePattern); + // set COLUMN_NAME + rowData.setString(3, rs.getString("Field")); + // set DATA_TYPE + String typeName = rs.getString("Type"); + rowData.setInt(4, getDataType(typeName)); + // set TYPE_NAME + rowData.setString(5, typeName); + // set COLUMN_SIZE + int length = rs.getInt("Length"); + rowData.setInt(6, getColumnSize(typeName, length)); + // set DECIMAL_DIGITS + rowData.setInt(8, getDecimalDigits(typeName)); + // set NUM_PREC_RADIX + rowData.setInt(9, 10); + // set NULLABLE + rowData.setInt(10, getNullable(index, typeName)); + // set REMARKS + rowData.setString(11, rs.getString("Note")); + rowDataList.add(rowData); + index++; + } + resultSet.setRowDataList(rowDataList); + return resultSet; + + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + } + + protected int getNullable(int index, String typeName) { + if (index == 0 && "TIMESTAMP".equals(typeName)) + return DatabaseMetaData.columnNoNulls; + return DatabaseMetaData.columnNullable; + } + + protected int getColumnSize(String typeName, int length) { + switch (typeName) { + case "TIMESTAMP": + return 23; + default: + return 0; + } + } + + protected int getDecimalDigits(String typeName) { + switch (typeName) { + case "FLOAT": + return 5; + case "DOUBLE": + return 9; + default: + return 0; + } + } + + protected int getDataType(String typeName) { + switch (typeName) { + case "TIMESTAMP": + return Types.TIMESTAMP; + case "INT": + return Types.INTEGER; + case "BIGINT": + return Types.BIGINT; + case "FLOAT": + return Types.FLOAT; + case "DOUBLE": + return Types.DOUBLE; + case "BINARY": + return Types.BINARY; + case "SMALLINT": + return Types.SMALLINT; + case "TINYINT": + return Types.TINYINT; + case "BOOL": + return Types.BOOLEAN; + case "NCHAR": + return Types.NCHAR; + default: + return Types.NULL; + } + } + + public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) + throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) + throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) + throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getVersionColumns(String catalog, String schema, String table) throws SQLException { + return getEmptyResultSet(); + } + + public abstract ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException; + + public ResultSet getImportedKeys(String catalog, String schema, String table) throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getExportedKeys(String catalog, String schema, String table) throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getCrossReference(String parentCatalog, String parentSchema, String parentTable, + String foreignCatalog, String foreignSchema, String foreignTable) throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getTypeInfo() throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) + throws SQLException { + return getEmptyResultSet(); + } + + public boolean supportsResultSetType(int type) throws SQLException { + return false; + } + + public boolean supportsResultSetConcurrency(int type, int concurrency) throws SQLException { + return false; + } + + public boolean ownUpdatesAreVisible(int type) throws SQLException { + return false; + } + + public boolean ownDeletesAreVisible(int type) throws SQLException { + return false; + } + + public boolean ownInsertsAreVisible(int type) throws SQLException { + return false; + } + + public boolean othersUpdatesAreVisible(int type) throws SQLException { + return false; + } + + public boolean othersDeletesAreVisible(int type) throws SQLException { + return false; + } + + public boolean othersInsertsAreVisible(int type) throws SQLException { + return false; + } + + public boolean updatesAreDetected(int type) throws SQLException { + return false; + } + + public boolean deletesAreDetected(int type) throws SQLException { + return false; + } + + public boolean insertsAreDetected(int type) throws SQLException { + return false; + } + + public boolean supportsBatchUpdates() throws SQLException { + return false; + } + + public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) + throws SQLException { + return getEmptyResultSet(); + } + + public abstract Connection getConnection() throws SQLException; + + public boolean supportsSavepoints() throws SQLException { + return false; + } + + public boolean supportsNamedParameters() throws SQLException { + return false; + } + + public boolean supportsMultipleOpenResults() throws SQLException { + return false; + } + + public boolean supportsGetGeneratedKeys() throws SQLException { + return false; + } + + public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) throws SQLException { + return getEmptyResultSet(); + } + + public abstract ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException; + + public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, + String attributeNamePattern) throws SQLException { + return getEmptyResultSet(); + } + + public boolean supportsResultSetHoldability(int holdability) throws SQLException { + if (holdability == ResultSet.HOLD_CURSORS_OVER_COMMIT) + return true; + return false; + } + + public int getResultSetHoldability() throws SQLException { + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + public int getDatabaseMajorVersion() throws SQLException { + return 2; + } + + public int getDatabaseMinorVersion() throws SQLException { + return 0; + } + + public int getJDBCMajorVersion() throws SQLException { + return 2; + } + + public int getJDBCMinorVersion() throws SQLException { + return 0; + } + + public int getSQLStateType() throws SQLException { + return 0; + } + + public boolean locatorsUpdateCopy() throws SQLException { + return false; + } + + public boolean supportsStatementPooling() throws SQLException { + return false; + } + + public RowIdLifetime getRowIdLifetime() throws SQLException { + return null; + } + + public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { + return getEmptyResultSet(); + } + + public boolean supportsStoredFunctionsUsingCallSyntax() throws SQLException { + return false; + } + + public boolean autoCommitFailureClosesAllResultSets() throws SQLException { + return false; + } + + public ResultSet getClientInfoProperties() throws SQLException { + //TODO: see https://docs.oracle.com/javase/8/docs/api/java/sql/Connection.html#setClientInfo-java.lang.String-java.lang.String- + return getEmptyResultSet(); + } + + public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) + throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getFunctionColumns(String catalog, String schemaPattern, String functionNamePattern, + String columnNamePattern) throws SQLException { + return getEmptyResultSet(); + } + + public ResultSet getPseudoColumns(String catalog, String schemaPattern, String tableNamePattern, + String columnNamePattern) throws SQLException { + return getEmptyResultSet(); + } + + public boolean generatedKeyAlwaysReturned() throws SQLException { + return false; + } + + private ResultSet getEmptyResultSet() { + return new EmptyResultSet(); + } + + protected ResultSet getCatalogs(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(1); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + + resultSet.setColumnMetaDataList(columnMetaDataList); + + List rowDataList = new ArrayList<>(); + ResultSet rs = stmt.executeQuery("show databases"); + while (rs.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(1); + rowData.setString(0, rs.getString("name")); + rowDataList.add(rowData); + } + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } + + + protected ResultSet getPrimaryKeys(String catalog, String schema, String table, Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(0); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + // TABLE_SCHEM + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(1); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + // TABLE_NAME + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(2); + col3.setColName("TABLE_NAME"); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + // COLUMN_NAME + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(3); + col4.setColName("COLUMN_NAME"); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + // KEY_SEQ + ColumnMetaData col5 = new ColumnMetaData(); + col5.setColIndex(4); + col5.setColName("KEY_SEQ"); + col5.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col5); + // PK_NAME + ColumnMetaData col6 = new ColumnMetaData(); + col6.setColIndex(5); + col6.setColName("PK_NAME"); + col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col6); + resultSet.setColumnMetaDataList(columnMetaDataList); + + // set rowData + List rowDataList = new ArrayList<>(); + ResultSet rs = stmt.executeQuery("describe " + dbname + "." + table); + rs.next(); + TSDBResultSetRowData rowData = new TSDBResultSetRowData(6); + rowData.setString(0, null); + rowData.setString(1, null); + rowData.setString(2, table); + String pkName = rs.getString(1); + rowData.setString(3, pkName); + rowData.setInt(4, 1); + rowData.setString(5, pkName); + rowDataList.add(rowData); + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } + + protected ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern, Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(0); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + // TABLE_SCHEM + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(1); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + // TABLE_NAME + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(2); + col3.setColName("TABLE_NAME"); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + // SUPERTABLE_NAME + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(3); + col4.setColName("SUPERTABLE_NAME"); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + resultSet.setColumnMetaDataList(columnMetaDataList); + + ResultSet rs = stmt.executeQuery("show tables like '" + tableNamePattern + "'"); + List rowDataList = new ArrayList<>(); + while (rs.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(4); + rowData.setString(2, rs.getString(1)); + rowData.setString(3, rs.getString(4)); + rowDataList.add(rowData); + } + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractTaosDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java similarity index 99% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractTaosDriver.java rename to src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java index f864788bfffc8bdfefb0b91ec645a10ae8eec843..21bf8e7a932b0515d77c8f124eae2d0a4596b3b6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractTaosDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java @@ -8,7 +8,7 @@ import java.util.List; import java.util.Properties; import java.util.StringTokenizer; -public abstract class AbstractTaosDriver implements Driver { +public abstract class AbstractDriver implements Driver { private static final String TAOS_CFG_FILENAME = "taos.cfg"; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java new file mode 100644 index 0000000000000000000000000000000000000000..14bd2929f17e344381510897528bc479c50a4d36 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java @@ -0,0 +1,1210 @@ +package com.taosdata.jdbc; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.*; +import java.util.Calendar; +import java.util.Map; + +public abstract class AbstractResultSet extends WrapperImpl implements ResultSet { + private int fetchSize; + + @Override + public abstract boolean next() throws SQLException; + + @Override + public abstract void close() throws SQLException; + + @Override + public boolean wasNull() throws SQLException { + return false; + } + + @Override + public abstract String getString(int columnIndex) throws SQLException; + + @Override + public abstract boolean getBoolean(int columnIndex) throws SQLException; + + @Override + public byte getByte(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract short getShort(int columnIndex) throws SQLException; + + @Override + public abstract int getInt(int columnIndex) throws SQLException; + + @Override + public abstract long getLong(int columnIndex) throws SQLException; + + @Override + public abstract float getFloat(int columnIndex) throws SQLException; + + @Override + public abstract double getDouble(int columnIndex) throws SQLException; + + @Override + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public byte[] getBytes(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract Timestamp getTimestamp(int columnIndex) throws SQLException; + + @Override + public InputStream getAsciiStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + } + + @Override + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public InputStream getBinaryStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public String getString(String columnLabel) throws SQLException { + return getString(findColumn(columnLabel)); + } + + @Override + public boolean getBoolean(String columnLabel) throws SQLException { + return getBoolean(findColumn(columnLabel)); + } + + @Override + public byte getByte(String columnLabel) throws SQLException { + return getByte(findColumn(columnLabel)); + } + + @Override + public short getShort(String columnLabel) throws SQLException { + return getShort(findColumn(columnLabel)); + } + + @Override + public int getInt(String columnLabel) throws SQLException { + return getInt(findColumn(columnLabel)); + } + + @Override + public long getLong(String columnLabel) throws SQLException { + return getLong(findColumn(columnLabel)); + } + + @Override + public float getFloat(String columnLabel) throws SQLException { + return getFloat(findColumn(columnLabel)); + } + + @Override + public double getDouble(String columnLabel) throws SQLException { + return getDouble(findColumn(columnLabel)); + } + + @Override + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + return getBigDecimal(findColumn(columnLabel)); + } + + @Override + public byte[] getBytes(String columnLabel) throws SQLException { + return getBytes(findColumn(columnLabel)); + } + + @Override + public Date getDate(String columnLabel) throws SQLException { + return getDate(findColumn(columnLabel)); + } + + @Override + public Time getTime(String columnLabel) throws SQLException { + return getTime(findColumn(columnLabel)); + } + + @Override + public Timestamp getTimestamp(String columnLabel) throws SQLException { + return getTimestamp(findColumn(columnLabel)); + } + + @Override + public InputStream getAsciiStream(String columnLabel) throws SQLException { + return getAsciiStream(findColumn(columnLabel)); + } + + @Override + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + return getUnicodeStream(findColumn(columnLabel)); + } + + @Override + public InputStream getBinaryStream(String columnLabel) throws SQLException { + return getBinaryStream(findColumn(columnLabel)); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + return null; + } + + @Override + public void clearWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + } + + @Override + public String getCursorName() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract ResultSetMetaData getMetaData() throws SQLException; + + @Override + public Object getObject(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Object getObject(String columnLabel) throws SQLException { + return getObject(findColumn(columnLabel)); + } + + @Override + public abstract int findColumn(String columnLabel) throws SQLException; + + @Override + public Reader getCharacterStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Reader getCharacterStream(String columnLabel) throws SQLException { + return getCharacterStream(findColumn(columnLabel)); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return getBigDecimal(findColumn(columnLabel)); + } + + @Override + public abstract boolean isBeforeFirst() throws SQLException; + + @Override + public abstract boolean isAfterLast() throws SQLException; + + @Override + public abstract boolean isFirst() throws SQLException; + + @Override + public abstract boolean isLast() throws SQLException; + + @Override + public abstract void beforeFirst() throws SQLException; + + @Override + public abstract void afterLast() throws SQLException; + + @Override + public abstract boolean first() throws SQLException; + + @Override + public abstract boolean last() throws SQLException; + + @Override + public abstract int getRow() throws SQLException; + + @Override + public abstract boolean absolute(int row) throws SQLException; + + @Override + public abstract boolean relative(int rows) throws SQLException; + + @Override + public abstract boolean previous() throws SQLException; + + @Override + public void setFetchDirection(int direction) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + //nothing to do + } + + @Override + public int getFetchDirection() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return ResultSet.FETCH_FORWARD; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (rows < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + //nothing to do + this.fetchSize = rows; + } + + @Override + public int getFetchSize() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return this.fetchSize; + } + + @Override + public int getType() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return ResultSet.TYPE_FORWARD_ONLY; + } + + @Override + public int getConcurrency() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return ResultSet.CONCUR_READ_ONLY; + } + + @Override + public boolean rowUpdated() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean rowInserted() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean rowDeleted() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNull(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBoolean(int columnIndex, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateByte(int columnIndex, byte x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateShort(int columnIndex, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateInt(int columnIndex, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateLong(int columnIndex, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateFloat(int columnIndex, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDouble(int columnIndex, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateString(int columnIndex, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBytes(int columnIndex, byte[] x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDate(int columnIndex, Date x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTime(int columnIndex, Time x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(int columnIndex, Object x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNull(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBoolean(String columnLabel, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateByte(String columnLabel, byte x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateShort(String columnLabel, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateInt(String columnLabel, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateLong(String columnLabel, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateFloat(String columnLabel, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDouble(String columnLabel, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateString(String columnLabel, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBytes(String columnLabel, byte[] x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDate(String columnLabel, Date x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTime(String columnLabel, Time x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(String columnLabel, Object x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void insertRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void deleteRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void refreshRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void cancelRowUpdates() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void moveToInsertRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void moveToCurrentRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract Statement getStatement() throws SQLException; + + @Override + public Object getObject(int columnIndex, Map> map) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Ref getRef(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Blob getBlob(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Clob getClob(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Array getArray(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Object getObject(String columnLabel, Map> map) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Ref getRef(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Blob getBlob(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Clob getClob(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Array getArray(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public URL getURL(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public URL getURL(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRef(int columnIndex, Ref x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRef(String columnLabel, Ref x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(int columnIndex, Blob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(String columnLabel, Blob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(int columnIndex, Clob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(String columnLabel, Clob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateArray(int columnIndex, Array x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateArray(String columnLabel, Array x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public RowId getRowId(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public RowId getRowId(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRowId(int columnIndex, RowId x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRowId(String columnLabel, RowId x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getHoldability() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public abstract boolean isClosed() throws SQLException; + + @Override + public void updateNString(int columnIndex, String nString) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNString(String columnLabel, String nString) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(int columnIndex, NClob nClob) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(String columnLabel, NClob nClob) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public NClob getNClob(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public NClob getNClob(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLXML getSQLXML(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLXML getSQLXML(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract String getNString(int columnIndex) throws SQLException; + + @Override + public String getNString(String columnLabel) throws SQLException { + return getNString(findColumn(columnLabel)); + } + + @Override + public Reader getNCharacterStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Reader getNCharacterStream(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(int columnIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(int columnIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public T getObject(int columnIndex, Class type) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public T getObject(String columnLabel, Class type) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java new file mode 100644 index 0000000000000000000000000000000000000000..8b6c074d1b859f179aac04a83cafb24cfae7c190 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java @@ -0,0 +1,326 @@ +package com.taosdata.jdbc; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractStatement extends WrapperImpl implements Statement { + + protected List batchedArgs; + private int fetchSize; + + @Override + public abstract ResultSet executeQuery(String sql) throws SQLException; + + @Override + public abstract int executeUpdate(String sql) throws SQLException; + + @Override + public abstract void close() throws SQLException; + + @Override + public int getMaxFieldSize() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return TSDBConstants.maxFieldSize; + } + + @Override + public void setMaxFieldSize(int max) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (max < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + // nothing to do + } + + @Override + public int getMaxRows() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return 0; + } + + @Override + public void setMaxRows(int max) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (max < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + // do nothing + } + + @Override + public void setEscapeProcessing(boolean enable) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // do nothing + } + + @Override + public int getQueryTimeout() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return 0; + } + + @Override + public void setQueryTimeout(int seconds) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (seconds < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + @Override + public void cancel() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return null; + } + + @Override + public void clearWarnings() throws SQLException { + // nothing to do + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } + + @Override + public void setCursorName(String name) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract boolean execute(String sql) throws SQLException; + + @Override + public abstract ResultSet getResultSet() throws SQLException; + + @Override + public abstract int getUpdateCount() throws SQLException; + + @Override + public boolean getMoreResults() throws SQLException { + return getMoreResults(CLOSE_CURRENT_RESULT); + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + switch (direction) { + case ResultSet.FETCH_FORWARD: + case ResultSet.FETCH_REVERSE: + case ResultSet.FETCH_UNKNOWN: + break; + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + //nothing to do + } + + @Override + public int getFetchDirection() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.FETCH_FORWARD; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (rows < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + //nothing to do + this.fetchSize = rows; + } + + @Override + public int getFetchSize() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return this.fetchSize; + } + + @Override + public int getResultSetConcurrency() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.CONCUR_READ_ONLY; + } + + @Override + public int getResultSetType() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.TYPE_FORWARD_ONLY; + } + + @Override + public void addBatch(String sql) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + if (batchedArgs == null) { + batchedArgs = new ArrayList<>(); + } + batchedArgs.add(sql); + } + + @Override + public void clearBatch() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (batchedArgs != null) + batchedArgs.clear(); + } + + @Override + public int[] executeBatch() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (batchedArgs == null || batchedArgs.isEmpty()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_BATCH_IS_EMPTY); + + int[] res = new int[batchedArgs.size()]; + for (int i = 0; i < batchedArgs.size(); i++) { + boolean isSelect = execute(batchedArgs.get(i)); + if (isSelect) { + res[i] = SUCCESS_NO_INFO; + } else { + res[i] = getUpdateCount(); + } + } + return res; + } + + @Override + public abstract Connection getConnection() throws SQLException; + + @Override + public boolean getMoreResults(int current) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + switch (current) { + case Statement.CLOSE_CURRENT_RESULT: + return false; + case Statement.KEEP_CURRENT_RESULT: + case Statement.CLOSE_ALL_RESULTS: + break; + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public ResultSet getGeneratedKeys() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int executeUpdate(String sql, String[] columnNames) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean execute(String sql, int[] columnIndexes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean execute(String sql, String[] columnNames) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getResultSetHoldability() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public abstract boolean isClosed() throws SQLException; + + @Override + public void setPoolable(boolean poolable) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // do nothing + } + + @Override + public boolean isPoolable() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return false; + } + + @Override + public void closeOnCompletion() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // do nothing + } + + @Override + public boolean isCloseOnCompletion() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return false; + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/CatalogResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/CatalogResultSet.java deleted file mode 100644 index 3a01e2e09297d6af1405c63200e2ba91b3fa99a2..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/CatalogResultSet.java +++ /dev/null @@ -1,68 +0,0 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; - -import java.sql.ResultSet; -import java.sql.SQLException; - -/* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the - * standard JDBC API contains more or less some adjustments customized for certain - * compatibility needs. - */ -public class CatalogResultSet extends TSDBResultSetWrapper { - - - public CatalogResultSet(ResultSet resultSet) { - super.setOriginalResultSet(resultSet); - } - - @Override - public String getString(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getString(columnIndex); - } else { - return null; - } - } - - @Override - public boolean getBoolean(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getBoolean(columnIndex); - } else { - return false; - } - } - - @Override - public byte[] getBytes(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getBytes(columnIndex); - } else { - return null; - } - } - - @Override - public Object getObject(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getObject(columnIndex); - } else { - return null; - } - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java index 633fdcd5ab7c9f077abbd725c2511bcc2251db44..fe16aa653546f4033bd944f0b6c72aec1863ab8a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java @@ -16,40 +16,40 @@ package com.taosdata.jdbc; public class ColumnMetaData { - private int colType = 0; - private String colName = null; - private int colSize = -1; - private int colIndex = 0; - - public int getColSize() { - return colSize; - } - - public void setColSize(int colSize) { - this.colSize = colSize; - } - - public int getColType() { - return colType; - } - - public void setColType(int colType) { - this.colType = colType; - } - - public String getColName() { - return colName; - } - - public void setColName(String colName) { - this.colName = colName; - } - - public int getColIndex() { - return colIndex; - } - - public void setColIndex(int colIndex) { - this.colIndex = colIndex; - } + private int colType = 0; + private String colName = null; + private int colSize = -1; + private int colIndex = 0; + + public int getColSize() { + return colSize; + } + + public void setColSize(int colSize) { + this.colSize = colSize; + } + + public int getColType() { + return colType; + } + + public void setColType(int colType) { + this.colType = colType; + } + + public String getColName() { + return colName; + } + + public void setColName(String colName) { + this.colName = colName; + } + + public int getColIndex() { + return colIndex; + } + + public void setColIndex(int colIndex) { + this.colIndex = colIndex; + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java index f82c064e751c195eb6327580c285a815346c917b..f6a0fcca316cb07d28c71a4e1d51d9405de083ba 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java @@ -160,12 +160,12 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public Date getDate(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public Time getTime(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -176,17 +176,17 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public InputStream getAsciiStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getUnicodeStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getBinaryStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -256,22 +256,22 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public InputStream getAsciiStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getUnicodeStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getBinaryStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -281,7 +281,7 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public String getCursorName() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -308,17 +308,17 @@ public class DatabaseMetaDataResultSet implements ResultSet { return colMetaData.getColIndex() + 1; } } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } @Override public Reader getCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public Reader getCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -353,22 +353,22 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public void beforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void afterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean first() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean last() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -383,17 +383,17 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public boolean absolute(int row) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean relative(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean previous() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -443,227 +443,227 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public void updateNull(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBoolean(int columnIndex, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateByte(int columnIndex, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateShort(int columnIndex, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateInt(int columnIndex, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateLong(int columnIndex, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateFloat(int columnIndex, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDouble(int columnIndex, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateString(int columnIndex, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBytes(int columnIndex, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDate(int columnIndex, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTime(int columnIndex, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(int columnIndex, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateNull(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBoolean(String columnLabel, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateByte(String columnLabel, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateShort(String columnLabel, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateInt(String columnLabel, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateLong(String columnLabel, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateFloat(String columnLabel, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDouble(String columnLabel, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateString(String columnLabel, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBytes(String columnLabel, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDate(String columnLabel, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTime(String columnLabel, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(String columnLabel, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void insertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void deleteRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void refreshRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void cancelRowUpdates() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void moveToInsertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void moveToCurrentRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -673,12 +673,12 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public Object getObject(int columnIndex, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public Ref getRef(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -1043,12 +1043,12 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetColumnsResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetColumnsResultSet.java deleted file mode 100644 index e15415e037948dd7ec757bcdeee03d14a0d588fb..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetColumnsResultSet.java +++ /dev/null @@ -1,51 +0,0 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; - -import java.sql.ResultSet; - -/* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the - * standard JDBC API contains more or less some adjustments customized for certain - * compatibility needs. - */ -public class GetColumnsResultSet extends TSDBResultSetWrapper { - private String catalog; - private String schemaPattern; - private String tableNamePattern; - private String columnNamePattern; - - public GetColumnsResultSet(ResultSet resultSet, String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) { - super.setOriginalResultSet(resultSet); - this.catalog = catalog; - this.schemaPattern = schemaPattern; - this.tableNamePattern = tableNamePattern; - this.columnNamePattern = columnNamePattern; - } - - @Override - public String getString(int columnIndex) { - switch (columnIndex) { - case 1: - return catalog; - case 2: - return null; - case 3: - return tableNamePattern; - default: - return null; - } - } -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetTablesResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetTablesResultSet.java deleted file mode 100644 index e28f6e3c9adf8564437e1214b28b2eb13bdaf8d9..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetTablesResultSet.java +++ /dev/null @@ -1,53 +0,0 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; - -import java.sql.ResultSet; -import java.sql.SQLException; - -/* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the - * standard JDBC API contains more or less some adjustments customized for certain - * compatibility needs. - */ -public class GetTablesResultSet extends TSDBResultSetWrapper { - - private String catalog; - private String schemaPattern; - private String tableNamePattern; - private String[] types; - - public GetTablesResultSet(ResultSet resultSet, String catalog, String schemaPattern, String tableNamePattern, String[] types) { - super.setOriginalResultSet(resultSet); - this.catalog = catalog; - this.schemaPattern = schemaPattern; - this.tableNamePattern = tableNamePattern; - this.types = types; - } - - @Override - public String getString(int columnIndex) throws SQLException { - String ret = null; - switch (columnIndex) { - case 3: - return super.getString(1); - case 4: - return "table"; - default: - return null; - } - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 94abe3965507170a4b31e17ebb431ddcb4fa11f8..8d947b9411eb91eded49b3c7b1f12586682346ff 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -14,65 +14,42 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.sql.Array; -import java.sql.Blob; -import java.sql.CallableStatement; -import java.sql.Clob; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.NClob; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLClientInfoException; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Savepoint; -import java.sql.Statement; -import java.sql.Struct; -import java.util.*; -import java.util.concurrent.Executor; +import java.sql.*; +import java.util.Properties; -public class TSDBConnection implements Connection { - protected Properties props = null; +public class TSDBConnection extends AbstractConnection { - private TSDBJNIConnector connector = null; + private TSDBJNIConnector connector; + private TSDBDatabaseMetaData databaseMetaData; + private boolean batchFetch; - private String catalog = null; - - private TSDBDatabaseMetaData dbMetaData = null; - - private Properties clientInfoProps = new Properties(); + public Boolean getBatchFetch() { + return this.batchFetch; + } - private int timeoutMilliseconds = 0; - - private boolean batchFetch = false; + public void setBatchFetch(Boolean batchFetch) { + this.batchFetch = batchFetch; + } public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { - this.dbMetaData = meta; + this.databaseMetaData = meta; connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), - info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), + info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), info.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD)); - + String batchLoad = info.getProperty(TSDBDriver.PROPERTY_KEY_BATCH_LOAD); if (batchLoad != null) { - this.batchFetch = Boolean.parseBoolean(batchLoad); + this.batchFetch = Boolean.parseBoolean(batchLoad); } } private void connect(String host, int port, String dbName, String user, String password) throws SQLException { this.connector = new TSDBJNIConnector(); this.connector.connect(host, port, dbName, user, password); - - try { - this.setCatalog(dbName); - } catch (SQLException e) { - e.printStackTrace(); - } - - this.dbMetaData.setConnection(this); + this.catalog = dbName; + this.databaseMetaData.setConnection(this); } public TSDBJNIConnector getConnection() { @@ -80,304 +57,59 @@ public class TSDBConnection implements Connection { } public Statement createStatement() throws SQLException { - if (!this.connector.isClosed()) { - TSDBStatement statement = new TSDBStatement(this, this.connector); - statement.setConnection(this); - return statement; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + + TSDBStatement statement = new TSDBStatement(this, this.connector); + statement.setConnection(this); + return statement; } public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { - if (this.connector.isClosed()) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } long id = this.connector.subscribe(topic, sql, restart, 0); if (id == 0) { - throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription")); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED); } - return new TSDBSubscribe(this.connector, id); } public PreparedStatement prepareStatement(String sql) throws SQLException { - if (!this.connector.isClosed()) { - return new TSDBPreparedStatement(this, this.connector, sql); - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } - } - - public CallableStatement prepareCall(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - public String nativeSQL(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setAutoCommit(boolean autoCommit) throws SQLException { - } - - public boolean getAutoCommit() throws SQLException { - return true; - } - - public void commit() throws SQLException { - } - - public void rollback() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return new TSDBPreparedStatement(this, this.connector, sql); } public void close() throws SQLException { - if (this.connector != null && !this.connector.isClosed()) { - this.connector.closeConnection(); - } else { - throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!")); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + this.isClosed = true; + this.connector.closeConnection(); } public boolean isClosed() throws SQLException { - return this.connector.isClosed(); + return this.connector != null && this.connector.isClosed(); } - /** - * A connection's database is able to provide information describing its tables, - * its supported SQL grammar, its stored procedures, the capabilities of this - * connection, etc. This information is made available through a - * DatabaseMetaData object. - * - * @return a DatabaseMetaData object for this connection - * @throws SQLException if a database access error occurs - */ public DatabaseMetaData getMetaData() throws SQLException { - return this.dbMetaData; - } - - /** - * This readOnly option is not supported by TDengine. However, the method is intentionally left blank here to - * support HikariCP connection. - * - * @param readOnly - * @throws SQLException - */ - public void setReadOnly(boolean readOnly) throws SQLException { - } - - public boolean isReadOnly() throws SQLException { - return true; - } - - public void setCatalog(String catalog) throws SQLException { - this.catalog = catalog; - } - - public String getCatalog() throws SQLException { - return this.catalog; - } - - /** - * The transaction isolation level option is not supported by TDengine. - * This method is intentionally left empty to support HikariCP connection. - * - * @param level - * @throws SQLException - */ - public void setTransactionIsolation(int level) throws SQLException { - } - - /** - * The transaction isolation level option is not supported by TDengine. - * - * @return - * @throws SQLException - */ - public int getTransactionIsolation() throws SQLException { - return Connection.TRANSACTION_NONE; - } - - public SQLWarning getWarnings() throws SQLException { - //todo: implement getWarnings according to the warning messages returned from TDengine - return null; -// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - // left blank to support HikariCP connection - //todo: implement clearWarnings according to the warning messages returned from TDengine - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) - throws SQLException { - // This method is implemented in the current way to support Spark - if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } - - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - return this.prepareStatement(sql); - } - - public Boolean getBatchFetch() { - return this.batchFetch; - } - - public void setBatchFetch(Boolean batchFetch) { - this.batchFetch = batchFetch; + return this.databaseMetaData; } - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Map> getTypeMap() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setTypeMap(Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setHoldability(int holdability) throws SQLException { - // intentionally left empty to support druid connection pool. - } - - /** - * the transaction is not supported by TDengine, so the opened ResultSet Objects will remain open - * - * @return - * @throws SQLException - */ - public int getHoldability() throws SQLException { - //intentionally left empty to support HikariCP connection. - return ResultSet.HOLD_CURSORS_OVER_COMMIT; - } - - public Savepoint setSavepoint() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Savepoint setSavepoint(String name) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void rollback(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void releaseSavepoint(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - return this.prepareStatement(sql, resultSetType, resultSetConcurrency); - } - - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob createClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob createBlob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob createNClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML createSQLXML() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isValid(int timeout) throws SQLException { - return !this.isClosed(); - } - - public void setClientInfo(String name, String value) throws SQLClientInfoException { - clientInfoProps.setProperty(name, value); - } - - public void setClientInfo(Properties properties) throws SQLClientInfoException { - for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { - String name = (String) enumer.nextElement(); - clientInfoProps.put(name, properties.getProperty(name)); + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } - public String getClientInfo(String name) throws SQLException { - return clientInfoProps.getProperty(name); - } - - public Properties getClientInfo() throws SQLException { - return clientInfoProps; - } - - public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setSchema(String schema) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getSchema() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void abort(Executor executor) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - this.timeoutMilliseconds = milliseconds; - } - - public int getNetworkTimeout() throws SQLException { - return this.timeoutMilliseconds; - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 3940e809300b7d8e0b6c79038afaa6ff76f81bf0..6179b47da1041a3461f1be43625f8c7d4e284e64 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -14,73 +14,138 @@ *****************************************************************************/ package com.taosdata.jdbc; +import java.sql.SQLException; +import java.sql.Types; import java.util.HashMap; import java.util.Map; public abstract class TSDBConstants { - public static final String DEFAULT_PORT = "6200"; - public static final String UNSUPPORT_METHOD_EXCEPTIONZ_MSG = "this operation is NOT supported currently!"; - public static final String INVALID_VARIABLES = "invalid variables"; - public static Map DATATYPE_MAP = null; + public static final String DEFAULT_PORT = "6200"; + public static Map DATATYPE_MAP = null; - public static final long JNI_NULL_POINTER = 0L; + public static final long JNI_NULL_POINTER = 0L; - public static final int JNI_SUCCESS = 0; - public static final int JNI_TDENGINE_ERROR = -1; - public static final int JNI_CONNECTION_NULL = -2; - public static final int JNI_RESULT_SET_NULL = -3; - public static final int JNI_NUM_OF_FIELDS_0 = -4; - public static final int JNI_SQL_NULL = -5; - public static final int JNI_FETCH_END = -6; - - public static final int TSDB_DATA_TYPE_NULL = 0; - public static final int TSDB_DATA_TYPE_BOOL = 1; - public static final int TSDB_DATA_TYPE_TINYINT = 2; - public static final int TSDB_DATA_TYPE_SMALLINT = 3; - public static final int TSDB_DATA_TYPE_INT = 4; - public static final int TSDB_DATA_TYPE_BIGINT = 5; - public static final int TSDB_DATA_TYPE_FLOAT = 6; - public static final int TSDB_DATA_TYPE_DOUBLE = 7; - public static final int TSDB_DATA_TYPE_BINARY = 8; - public static final int TSDB_DATA_TYPE_TIMESTAMP = 9; - public static final int TSDB_DATA_TYPE_NCHAR = 10; - - public static String WrapErrMsg(String msg) { - return "TDengine Error: " + msg; - } + public static final int JNI_SUCCESS = 0; + public static final int JNI_TDENGINE_ERROR = -1; + public static final int JNI_CONNECTION_NULL = -2; + public static final int JNI_RESULT_SET_NULL = -3; + public static final int JNI_NUM_OF_FIELDS_0 = -4; + public static final int JNI_SQL_NULL = -5; + public static final int JNI_FETCH_END = -6; + public static final int JNI_OUT_OF_MEMORY = -7; - public static String FixErrMsg(int code) { - switch (code) { - case JNI_TDENGINE_ERROR: - return WrapErrMsg("internal error of database!"); - case JNI_CONNECTION_NULL: - return WrapErrMsg("invalid tdengine connection!"); - case JNI_RESULT_SET_NULL: - return WrapErrMsg("invalid resultset pointer!"); - case JNI_NUM_OF_FIELDS_0: - return WrapErrMsg("invalid num of fields!"); - case JNI_SQL_NULL: - return WrapErrMsg("can't execute empty sql!"); - case JNI_FETCH_END: - return WrapErrMsg("fetch to the end of resultset"); - default: - break; - } - return WrapErrMsg("unkown error!"); - } + public static final int TSDB_DATA_TYPE_NULL = 0; + public static final int TSDB_DATA_TYPE_BOOL = 1; + public static final int TSDB_DATA_TYPE_TINYINT = 2; + public static final int TSDB_DATA_TYPE_SMALLINT = 3; + public static final int TSDB_DATA_TYPE_INT = 4; + public static final int TSDB_DATA_TYPE_BIGINT = 5; + public static final int TSDB_DATA_TYPE_FLOAT = 6; + public static final int TSDB_DATA_TYPE_DOUBLE = 7; + public static final int TSDB_DATA_TYPE_BINARY = 8; + public static final int TSDB_DATA_TYPE_TIMESTAMP = 9; + public static final int TSDB_DATA_TYPE_NCHAR = 10; - static { - DATATYPE_MAP = new HashMap(); - DATATYPE_MAP.put(1, "BOOL"); - DATATYPE_MAP.put(2, "TINYINT"); - DATATYPE_MAP.put(3, "SMALLINT"); - DATATYPE_MAP.put(4, "INT"); - DATATYPE_MAP.put(5, "BIGINT"); - DATATYPE_MAP.put(6, "FLOAT"); - DATATYPE_MAP.put(7, "DOUBLE"); - DATATYPE_MAP.put(8, "BINARY"); - DATATYPE_MAP.put(9, "TIMESTAMP"); - DATATYPE_MAP.put(10, "NCHAR"); - } + // nchar field's max length + public static final int maxFieldSize = 16 * 1024; + + public static String WrapErrMsg(String msg) { + return "TDengine Error: " + msg; + } + + public static String FixErrMsg(int code) { + switch (code) { + case JNI_TDENGINE_ERROR: + return WrapErrMsg("internal error of database!"); + case JNI_CONNECTION_NULL: + return WrapErrMsg("invalid tdengine connection!"); + case JNI_RESULT_SET_NULL: + return WrapErrMsg("invalid resultset pointer!"); + case JNI_NUM_OF_FIELDS_0: + return WrapErrMsg("invalid num of fields!"); + case JNI_SQL_NULL: + return WrapErrMsg("can't execute empty sql!"); + case JNI_FETCH_END: + return WrapErrMsg("fetch to the end of resultset"); + default: + break; + } + return WrapErrMsg("unkown error!"); + } + + public static int taosType2JdbcType(int taosType) throws SQLException { + switch (taosType) { + case TSDBConstants.TSDB_DATA_TYPE_NULL: + return Types.NULL; + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return Types.BOOLEAN; + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return Types.TINYINT; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return Types.SMALLINT; + case TSDBConstants.TSDB_DATA_TYPE_INT: + return Types.INTEGER; + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return Types.BIGINT; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return Types.FLOAT; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return Types.DOUBLE; + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return Types.BINARY; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return Types.TIMESTAMP; + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return Types.NCHAR; + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); + } + + public static int jdbcType2TaosType(int jdbcType) throws SQLException { + switch (jdbcType){ + case Types.NULL: + return TSDBConstants.TSDB_DATA_TYPE_NULL; + case Types.BOOLEAN: + return TSDBConstants.TSDB_DATA_TYPE_BOOL; + case Types.TINYINT: + return TSDBConstants.TSDB_DATA_TYPE_TINYINT; + case Types.SMALLINT: + return TSDBConstants.TSDB_DATA_TYPE_SMALLINT; + case Types.INTEGER: + return TSDBConstants.TSDB_DATA_TYPE_INT; + case Types.BIGINT: + return TSDBConstants.TSDB_DATA_TYPE_BIGINT; + case Types.FLOAT: + return TSDBConstants.TSDB_DATA_TYPE_FLOAT; + case Types.DOUBLE: + return TSDBConstants.TSDB_DATA_TYPE_DOUBLE; + case Types.BINARY: + return TSDBConstants.TSDB_DATA_TYPE_BINARY; + case Types.TIMESTAMP: + return TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP; + case Types.NCHAR: + return TSDBConstants.TSDB_DATA_TYPE_NCHAR; + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); + } + + static { + DATATYPE_MAP = new HashMap<>(); + DATATYPE_MAP.put(0, "NULL"); + DATATYPE_MAP.put(1, "BOOL"); + DATATYPE_MAP.put(2, "TINYINT"); + DATATYPE_MAP.put(3, "SMALLINT"); + DATATYPE_MAP.put(4, "INT"); + DATATYPE_MAP.put(5, "BIGINT"); + DATATYPE_MAP.put(6, "FLOAT"); + DATATYPE_MAP.put(7, "DOUBLE"); + DATATYPE_MAP.put(8, "BINARY"); + DATATYPE_MAP.put(9, "TIMESTAMP"); + DATATYPE_MAP.put(10, "NCHAR"); + } + + public static String jdbcType2TaosTypeName(int type) throws SQLException { + return DATATYPE_MAP.get(jdbcType2TaosType(type)); + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java index f4dee67adf03a3474f582dbe662a8c5e988e3fab..8b7ede148e89cce0d8db22e62627bd1e1c49f9bb 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java @@ -14,41 +14,27 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; -public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData { +public class TSDBDatabaseMetaData extends AbstractDatabaseMetaData { - private String dbProductName = null; - private String url = null; - private String userName = null; - private Connection conn = null; + private String url; + private String userName; + private Connection conn; - public TSDBDatabaseMetaData(String dbProductName, String url, String userName) { - this.dbProductName = dbProductName; + public TSDBDatabaseMetaData(String url, String userName) { this.url = url; this.userName = userName; } - public void setConnection(Connection conn) { - this.conn = conn; - } - - public T unwrap(Class iface) throws SQLException { - return null; - } - - public boolean isWrapperFor(Class iface) throws SQLException { - return false; - } - - public boolean allProceduresAreCallable() throws SQLException { - return false; + public Connection getConnection() throws SQLException { + return this.conn; } - public boolean allTablesAreSelectable() throws SQLException { - return false; + public void setConnection(Connection conn) { + this.conn = conn; } public String getURL() throws SQLException { @@ -59,906 +45,51 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData { return this.userName; } - public boolean isReadOnly() throws SQLException { - return false; - } - - public boolean nullsAreSortedHigh() throws SQLException { - return false; - } - - public boolean nullsAreSortedLow() throws SQLException { - return !nullsAreSortedHigh(); - } - - public boolean nullsAreSortedAtStart() throws SQLException { - return true; - } - - public boolean nullsAreSortedAtEnd() throws SQLException { - return !nullsAreSortedAtStart(); - } - - public String getDatabaseProductName() throws SQLException { - return this.dbProductName; - } - - public String getDatabaseProductVersion() throws SQLException { - return "1.5.1"; - } - public String getDriverName() throws SQLException { return TSDBDriver.class.getName(); } - public String getDriverVersion() throws SQLException { - return "1.0.0"; - } - - public int getDriverMajorVersion() { - return 2; - } - - public int getDriverMinorVersion() { - return 0; - } - - public boolean usesLocalFiles() throws SQLException { - return false; - } - - public boolean usesLocalFilePerTable() throws SQLException { - return false; - } - - public boolean supportsMixedCaseIdentifiers() throws SQLException { - return false; - } - - public boolean storesUpperCaseIdentifiers() throws SQLException { - return false; - } - - public boolean storesLowerCaseIdentifiers() throws SQLException { - return false; - } - - public boolean storesMixedCaseIdentifiers() throws SQLException { - return false; - } - - public boolean supportsMixedCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public boolean storesUpperCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public boolean storesLowerCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public boolean storesMixedCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public String getIdentifierQuoteString() throws SQLException { - return " "; - } - - public String getSQLKeywords() throws SQLException { - return null; - } - - public String getNumericFunctions() throws SQLException { - return null; - } - - public String getStringFunctions() throws SQLException { - return null; - } - - public String getSystemFunctions() throws SQLException { - return null; - } - - public String getTimeDateFunctions() throws SQLException { - return null; - } - - public String getSearchStringEscape() throws SQLException { - return null; - } - - public String getExtraNameCharacters() throws SQLException { - return null; - } - - public boolean supportsAlterTableWithAddColumn() throws SQLException { - return true; - } - - public boolean supportsAlterTableWithDropColumn() throws SQLException { - return true; - } - - public boolean supportsColumnAliasing() throws SQLException { - return true; - } - - public boolean nullPlusNonNullIsNull() throws SQLException { - return false; - } - - public boolean supportsConvert() throws SQLException { - return false; - } - - public boolean supportsConvert(int fromType, int toType) throws SQLException { - return false; - } - - public boolean supportsTableCorrelationNames() throws SQLException { - return false; - } - - public boolean supportsDifferentTableCorrelationNames() throws SQLException { - return false; - } - - public boolean supportsExpressionsInOrderBy() throws SQLException { - return false; - } - - public boolean supportsOrderByUnrelated() throws SQLException { - return false; - } - - public boolean supportsGroupBy() throws SQLException { - return false; - } - - public boolean supportsGroupByUnrelated() throws SQLException { - return false; - } - - public boolean supportsGroupByBeyondSelect() throws SQLException { - return false; - } - - public boolean supportsLikeEscapeClause() throws SQLException { - return false; - } - - public boolean supportsMultipleResultSets() throws SQLException { - return false; - } - - public boolean supportsMultipleTransactions() throws SQLException { - return false; - } - - public boolean supportsNonNullableColumns() throws SQLException { - return false; - } - - public boolean supportsMinimumSQLGrammar() throws SQLException { - return false; - } - - public boolean supportsCoreSQLGrammar() throws SQLException { - return false; - } - - public boolean supportsExtendedSQLGrammar() throws SQLException { - return false; - } - - public boolean supportsANSI92EntryLevelSQL() throws SQLException { - return false; - } - - public boolean supportsANSI92IntermediateSQL() throws SQLException { - return false; - } - - public boolean supportsANSI92FullSQL() throws SQLException { - return false; - } - - public boolean supportsIntegrityEnhancementFacility() throws SQLException { - return false; - } - - public boolean supportsOuterJoins() throws SQLException { - return false; - } - - public boolean supportsFullOuterJoins() throws SQLException { - return false; - } - - public boolean supportsLimitedOuterJoins() throws SQLException { - return false; - } - - public String getSchemaTerm() throws SQLException { - return null; - } - - public String getProcedureTerm() throws SQLException { - return null; - } - - public String getCatalogTerm() throws SQLException { - return "database"; - } - - public boolean isCatalogAtStart() throws SQLException { - return true; - } - - public String getCatalogSeparator() throws SQLException { - return "."; - } - - public boolean supportsSchemasInDataManipulation() throws SQLException { - return false; - } - - public boolean supportsSchemasInProcedureCalls() throws SQLException { - return false; - } - - public boolean supportsSchemasInTableDefinitions() throws SQLException { - return false; - } - - public boolean supportsSchemasInIndexDefinitions() throws SQLException { - return false; - } - - public boolean supportsSchemasInPrivilegeDefinitions() throws SQLException { - return false; - } - - public boolean supportsCatalogsInDataManipulation() throws SQLException { - return true; - } - - public boolean supportsCatalogsInProcedureCalls() throws SQLException { - return false; - } - - public boolean supportsCatalogsInTableDefinitions() throws SQLException { - return false; - } - - public boolean supportsCatalogsInIndexDefinitions() throws SQLException { - return false; - } - - public boolean supportsCatalogsInPrivilegeDefinitions() throws SQLException { - return false; - } - - public boolean supportsPositionedDelete() throws SQLException { - return false; - } - - public boolean supportsPositionedUpdate() throws SQLException { - return false; - } - - public boolean supportsSelectForUpdate() throws SQLException { - return false; - } - - public boolean supportsStoredProcedures() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInComparisons() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInExists() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInIns() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInQuantifieds() throws SQLException { - return false; - } - - public boolean supportsCorrelatedSubqueries() throws SQLException { - return false; - } - - public boolean supportsUnion() throws SQLException { - return false; - } - - public boolean supportsUnionAll() throws SQLException { - return false; - } - - public boolean supportsOpenCursorsAcrossCommit() throws SQLException { - return false; - } - - public boolean supportsOpenCursorsAcrossRollback() throws SQLException { - return false; - } - - public boolean supportsOpenStatementsAcrossCommit() throws SQLException { - return false; - } - - public boolean supportsOpenStatementsAcrossRollback() throws SQLException { - return false; - } - - public int getMaxBinaryLiteralLength() throws SQLException { - return 0; - } - - public int getMaxCharLiteralLength() throws SQLException { - return 0; - } - - public int getMaxColumnNameLength() throws SQLException { - return 0; - } - - public int getMaxColumnsInGroupBy() throws SQLException { - return 0; - } - - public int getMaxColumnsInIndex() throws SQLException { - return 0; - } - - public int getMaxColumnsInOrderBy() throws SQLException { - return 0; - } - - public int getMaxColumnsInSelect() throws SQLException { - return 0; - } - - public int getMaxColumnsInTable() throws SQLException { - return 0; - } - - public int getMaxConnections() throws SQLException { - return 0; - } - - public int getMaxCursorNameLength() throws SQLException { - return 0; - } - - public int getMaxIndexLength() throws SQLException { - return 0; - } - - public int getMaxSchemaNameLength() throws SQLException { - return 0; - } - - public int getMaxProcedureNameLength() throws SQLException { - return 0; - } - - public int getMaxCatalogNameLength() throws SQLException { - return 0; - } - - public int getMaxRowSize() throws SQLException { - return 0; - } - - public boolean doesMaxRowSizeIncludeBlobs() throws SQLException { - return false; - } - - public int getMaxStatementLength() throws SQLException { - return 0; - } - - public int getMaxStatements() throws SQLException { - return 0; - } - - public int getMaxTableNameLength() throws SQLException { - return 0; - } - - public int getMaxTablesInSelect() throws SQLException { - return 0; - } - - public int getMaxUserNameLength() throws SQLException { - return 0; - } - - public int getDefaultTransactionIsolation() throws SQLException { - return 0; - } - - public boolean supportsTransactions() throws SQLException { - return false; - } - - public boolean supportsTransactionIsolationLevel(int level) throws SQLException { - return false; - } - - public boolean supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { - return false; - } - - public boolean supportsDataManipulationTransactionsOnly() throws SQLException { - return false; - } - - public boolean dataDefinitionCausesTransactionCommit() throws SQLException { - return false; - } - - public boolean dataDefinitionIgnoredInTransactions() throws SQLException { - return false; - } - - public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) - throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, - String columnNamePattern) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) - throws SQLException { - Statement stmt = null; - if (null != conn && !conn.isClosed()) { - stmt = conn.createStatement(); - if (catalog == null || catalog.length() < 1) { - catalog = conn.getCatalog(); - } - stmt.executeUpdate("use " + catalog); - ResultSet resultSet0 = stmt.executeQuery("show tables"); - GetTablesResultSet getTablesResultSet = new GetTablesResultSet(resultSet0, catalog, schemaPattern, tableNamePattern, types); - return getTablesResultSet; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + /** + * @Param catalog : database名称，"" 表示不属于任何database的table，null表示不使用database来缩小范围 + * @Param schemaPattern : schema名称，""表示 + * @Param tableNamePattern : 表名满足tableNamePattern的表, null表示返回所有表 + * @Param types : 表类型，null表示返回所有类型 + */ + public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { + if (conn == null || conn.isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } - } - - public ResultSet getSchemas() throws SQLException { - return getEmptyResultSet(); + return super.getTables(catalog, schemaPattern, tableNamePattern, types, conn); } public ResultSet getCatalogs() throws SQLException { - - if (conn != null && !conn.isClosed()) { - Statement stmt = conn.createStatement(); - ResultSet resultSet0 = stmt.executeQuery("show databases"); - CatalogResultSet resultSet = new CatalogResultSet(resultSet0); - return resultSet; - } else { - return getEmptyResultSet(); - } + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getCatalogs(conn); } public ResultSet getTableTypes() throws SQLException { - DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - - // set up ColumnMetaDataList - List columnMetaDataList = new ArrayList(1); - ColumnMetaData colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(0); - colMetaData.setColName("TABLE_TYPE"); - colMetaData.setColSize(10); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - - // set up rowDataList - List rowDataList = new ArrayList(2); - TSDBResultSetRowData rowData = new TSDBResultSetRowData(); - rowData.setString(0, "TABLE"); - rowDataList.add(rowData); - rowData = new TSDBResultSetRowData(); - rowData.setString(0, "STABLE"); - rowDataList.add(rowData); - - resultSet.setColumnMetaDataList(columnMetaDataList); - resultSet.setRowDataList(rowDataList); - return resultSet; - } - - public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) - throws SQLException { - - /** add by zyyang **********/ - Statement stmt = null; - if (null != conn && !conn.isClosed()) { - stmt = conn.createStatement(); - if (catalog == null || catalog.length() < 1) { - catalog = conn.getCatalog(); - } - stmt.executeUpdate("use " + catalog); - - DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - // set up ColumnMetaDataList - List columnMetaDataList = new ArrayList<>(24); - columnMetaDataList.add(null); - columnMetaDataList.add(null); - // add TABLE_NAME - ColumnMetaData colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(3); - colMetaData.setColName("TABLE_NAME"); - colMetaData.setColSize(193); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add COLUMN_NAME - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(4); - colMetaData.setColName("COLUMN_NAME"); - colMetaData.setColSize(65); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add DATA_TYPE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(5); - colMetaData.setColName("DATA_TYPE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add TYPE_NAME - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(6); - colMetaData.setColName("TYPE_NAME"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add COLUMN_SIZE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(7); - colMetaData.setColName("COLUMN_SIZE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add BUFFER_LENGTH ,not used - columnMetaDataList.add(null); - // add DECIMAL_DIGITS - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(9); - colMetaData.setColName("DECIMAL_DIGITS"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add NUM_PREC_RADIX - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(10); - colMetaData.setColName("NUM_PREC_RADIX"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add NULLABLE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(11); - colMetaData.setColName("NULLABLE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - - resultSet.setColumnMetaDataList(columnMetaDataList); - - // set up rowDataList - ResultSet resultSet0 = stmt.executeQuery("describe " + tableNamePattern); - List rowDataList = new ArrayList<>(); - int index = 0; - while (resultSet0.next()) { - TSDBResultSetRowData rowData = new TSDBResultSetRowData(24); - // set TABLE_NAME - rowData.setString(2, tableNamePattern); - // set COLUMN_NAME - rowData.setString(3, resultSet0.getString(1)); - // set DATA_TYPE - String typeName = resultSet0.getString(2); - rowData.setInt(4, getDataType(typeName)); - // set TYPE_NAME - rowData.setString(5, typeName); - // set COLUMN_SIZE - int length = resultSet0.getInt(3); - rowData.setInt(6, getColumnSize(typeName, length)); - // set DECIMAL_DIGITS - rowData.setInt(8, getDecimalDigits(typeName)); - // set NUM_PREC_RADIX - rowData.setInt(9, 10); - // set NULLABLE - rowData.setInt(10, getNullable(index, typeName)); - rowDataList.add(rowData); - index++; - } - resultSet.setRowDataList(rowDataList); - -// GetColumnsResultSet getColumnsResultSet = new GetColumnsResultSet(resultSet0, catalog, schemaPattern, tableNamePattern, columnNamePattern); -// return getColumnsResultSet; -// DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - return resultSet; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - - /*************************/ - -// return getEmptyResultSet(); - } - - private int getNullable(int index, String typeName) { - if (index == 0 && "TIMESTAMP".equals(typeName)) - return DatabaseMetaData.columnNoNulls; - return DatabaseMetaData.columnNullable; - } - - private int getColumnSize(String typeName, int length) { - switch (typeName) { - case "TIMESTAMP": - return 23; - - default: - return 0; - } - } - - private int getDecimalDigits(String typeName) { - switch (typeName) { - case "FLOAT": - return 5; - case "DOUBLE": - return 9; - default: - return 0; - } - } - - private int getDataType(String typeName) { - switch (typeName) { - case "TIMESTAMP": - return Types.TIMESTAMP; - case "INT": - return Types.INTEGER; - case "BIGINT": - return Types.BIGINT; - case "FLOAT": - return Types.FLOAT; - case "DOUBLE": - return Types.DOUBLE; - case "BINARY": - return Types.BINARY; - case "SMALLINT": - return Types.SMALLINT; - case "TINYINT": - return Types.TINYINT; - case "BOOL": - return Types.BOOLEAN; - case "NCHAR": - return Types.NCHAR; - default: - return Types.NULL; - } - } - - public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) - throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) - throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) - throws SQLException { - return getEmptyResultSet(); + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getTableTypes(); } - public ResultSet getVersionColumns(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); + public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern, conn); } public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getImportedKeys(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getExportedKeys(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getCrossReference(String parentCatalog, String parentSchema, String parentTable, - String foreignCatalog, String foreignSchema, String foreignTable) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getTypeInfo() throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) - throws SQLException { - return getEmptyResultSet(); - } - - public boolean supportsResultSetType(int type) throws SQLException { - return false; - } - - public boolean supportsResultSetConcurrency(int type, int concurrency) throws SQLException { - return false; - } - - public boolean ownUpdatesAreVisible(int type) throws SQLException { - return false; - } - - public boolean ownDeletesAreVisible(int type) throws SQLException { - return false; - } - - public boolean ownInsertsAreVisible(int type) throws SQLException { - return false; - } - - public boolean othersUpdatesAreVisible(int type) throws SQLException { - return false; - } - - public boolean othersDeletesAreVisible(int type) throws SQLException { - return false; - } - - public boolean othersInsertsAreVisible(int type) throws SQLException { - return false; - } - - public boolean updatesAreDetected(int type) throws SQLException { - return false; - } - - public boolean deletesAreDetected(int type) throws SQLException { - return false; - } - - public boolean insertsAreDetected(int type) throws SQLException { - return false; - } - - public boolean supportsBatchUpdates() throws SQLException { - return false; - } - - public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) - throws SQLException { - return getEmptyResultSet(); - } - - public Connection getConnection() throws SQLException { - return null; - } - - public boolean supportsSavepoints() throws SQLException { - return false; - } - - public boolean supportsNamedParameters() throws SQLException { - return false; - } - - public boolean supportsMultipleOpenResults() throws SQLException { - return false; - } - - public boolean supportsGetGeneratedKeys() throws SQLException { - return false; - } - - public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) throws SQLException { - return getEmptyResultSet(); + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getPrimaryKeys(catalog, schema, table, conn); } public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, - String attributeNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public boolean supportsResultSetHoldability(int holdability) throws SQLException { - return false; - } - - public int getResultSetHoldability() throws SQLException { - return 0; + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getSuperTables(catalog, schemaPattern, tableNamePattern, conn); } - public int getDatabaseMajorVersion() throws SQLException { - return 0; - } - - public int getDatabaseMinorVersion() throws SQLException { - return 0; - } - - public int getJDBCMajorVersion() throws SQLException { - return 0; - } - - public int getJDBCMinorVersion() throws SQLException { - return 0; - } - - public int getSQLStateType() throws SQLException { - return 0; - } - - public boolean locatorsUpdateCopy() throws SQLException { - return false; - } - - public boolean supportsStatementPooling() throws SQLException { - return false; - } - - public RowIdLifetime getRowIdLifetime() throws SQLException { - return null; - } - - public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { - return null; - } - - public boolean supportsStoredFunctionsUsingCallSyntax() throws SQLException { - return false; - } - - public boolean autoCommitFailureClosesAllResultSets() throws SQLException { - return false; - } - - public ResultSet getClientInfoProperties() throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) - throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getFunctionColumns(String catalog, String schemaPattern, String functionNamePattern, - String columnNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getPseudoColumns(String catalog, String schemaPattern, String tableNamePattern, - String columnNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public boolean generatedKeyAlwaysReturned() throws SQLException { - return false; - } - - private ResultSet getEmptyResultSet() { - return new EmptyResultSet(); - } } \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 06f88cebfaa8aa90cc81506a98374ec8076ad82e..2b87b72fef0f2b621536c5a11aba69975aa86434 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -37,7 +37,7 @@ import java.util.logging.Logger; * register it with the DriverManager. This means that a user can load and * register a driver by doing Class.forName("foo.bah.Driver") */ -public class TSDBDriver extends AbstractTaosDriver { +public class TSDBDriver extends AbstractDriver { @Deprecated private static final String URL_PREFIX1 = "jdbc:TSDB://"; @@ -90,7 +90,7 @@ public class TSDBDriver extends AbstractTaosDriver { * fetch data from native function in a batch model */ public static final String PROPERTY_KEY_BATCH_LOAD = "batchfetch"; - + private TSDBDatabaseMetaData dbMetaData = null; static { @@ -179,18 +179,18 @@ public class TSDBDriver extends AbstractTaosDriver { while (queryParams.hasMoreElements()) { String oneToken = queryParams.nextToken(); String[] pair = oneToken.split("="); - + if ((pair[0] != null && pair[0].trim().length() > 0) && (pair[1] != null && pair[1].trim().length() > 0)) { urlProps.setProperty(pair[0].trim(), pair[1].trim()); } } } - + // parse Product Name String dbProductName = url.substring(0, beginningOfSlashes); dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1); dbProductName = dbProductName.substring(0, dbProductName.indexOf(":")); - + // parse database name url = url.substring(beginningOfSlashes + 2); int indexOfSlash = url.indexOf("/"); @@ -200,7 +200,7 @@ public class TSDBDriver extends AbstractTaosDriver { } url = url.substring(0, indexOfSlash); } - + // parse port int indexOfColon = url.indexOf(":"); if (indexOfColon != -1) { @@ -209,12 +209,12 @@ public class TSDBDriver extends AbstractTaosDriver { } url = url.substring(0, indexOfColon); } - + if (url != null && url.length() > 0 && url.trim().length() > 0) { urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url); } - - this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER)); + + this.dbMetaData = new TSDBDatabaseMetaData(urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER)); return urlProps; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java new file mode 100644 index 0000000000000000000000000000000000000000..c7717e331b39247542d1a022cafd1cab2ac65627 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java @@ -0,0 +1,68 @@ +package com.taosdata.jdbc; + +import java.sql.SQLClientInfoException; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.HashMap; +import java.util.Map; + +public class TSDBError { + private static Map TSDBErrorMap = new HashMap<>(); + + static { + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED, "connection already closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD, "this operation is NOT supported currently!"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variables"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED, "statement is closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED, "resultSet is closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_BATCH_IS_EMPTY, "Batch is empty!"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY, "Can not issue data manipulation statements with executeQuery()"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEUPDATE, "Can not issue SELECT via executeUpdate()"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: (?)"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE, "Database not specified or available"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: (?)"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "not a valid sql for execute: (?)"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "parameter index out of range"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED, "connection already closed"); + /**************************************************/ + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error"); + /**************************************************/ + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED, "failed to create subscription"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_ENCODING, "Unsupported encoding"); + + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_TDENGINE_ERROR, "internal error of database"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL, "JNI connection is NULL"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL, "JNI result set is NULL"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0, "invalid num of fields"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_SQL_NULL, "empty sql string"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_FETCH_END, "fetch to the end of resultSet"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_OUT_OF_MEMORY, "JNI alloc memory failed"); + } + + public static SQLException createSQLException(int errorCode) { + String message; + if (TSDBErrorNumbers.contains(errorCode)) + message = TSDBErrorMap.get(errorCode); + else + message = TSDBErrorMap.get(TSDBErrorNumbers.ERROR_UNKNOWN); + return createSQLException(errorCode, message); + } + + public static SQLException createSQLException(int errorCode, String message) { + // throw SQLFeatureNotSupportedException + if (errorCode == TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD) + return new SQLFeatureNotSupportedException(message, "", errorCode); + // throw SQLClientInfoException + if (errorCode == TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED) + return new SQLClientInfoException(message, null); + + if (errorCode > 0x2300 && errorCode < 0x2350) + // JDBC exception's error number is less than 0x2350 + return new SQLException("ERROR (" + Integer.toHexString(errorCode) + "): " + message, "", errorCode); + if (errorCode > 0x2350 && errorCode < 0x2400) + // JNI exception's error number is large than 0x2350 + return new SQLException("JNI ERROR (" + Integer.toHexString(errorCode) + "): " + message, "", errorCode); + return new SQLException("TDengine ERROR (" + Integer.toHexString(errorCode) + "): " + message, "", errorCode); + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java new file mode 100644 index 0000000000000000000000000000000000000000..78e7aec79c252a2e6b2eeafcba37ab1c66f8674d --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -0,0 +1,75 @@ +package com.taosdata.jdbc; + +import java.util.HashSet; + +public class TSDBErrorNumbers { + + public static final int ERROR_CONNECTION_CLOSED = 0x2301; // connection already closed + public static final int ERROR_UNSUPPORTED_METHOD = 0x2302; //this operation is NOT supported currently! + public static final int ERROR_INVALID_VARIABLE = 0x2303; //invalid variables + public static final int ERROR_STATEMENT_CLOSED = 0x2304; //statement already closed + public static final int ERROR_RESULTSET_CLOSED = 0x2305; //resultSet is closed + public static final int ERROR_BATCH_IS_EMPTY = 0x2306; //Batch is empty! + public static final int ERROR_INVALID_WITH_EXECUTEQUERY = 0x2307; //Can not issue data manipulation statements with executeQuery() + public static final int ERROR_INVALID_WITH_EXECUTEUPDATE = 0x2308; //Can not issue SELECT via executeUpdate() + public static final int ERROR_INVALID_FOR_EXECUTE_QUERY = 0x2309; //not a valid sql for executeQuery: (SQL) + public static final int ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE = 0x230a; //Database not specified or available + public static final int ERROR_INVALID_FOR_EXECUTE_UPDATE = 0x230b; //not a valid sql for executeUpdate: (SQL) + public static final int ERROR_INVALID_FOR_EXECUTE = 0x230c; //not a valid sql for execute: (SQL) + public static final int ERROR_PARAMETER_INDEX_OUT_RANGE = 0x230d; // parameter index out of range + public static final int ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED = 0x230e; // connection already closed + public static final int ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE = 0x230f; //unknown sql type in tdengine + + public static final int ERROR_UNKNOWN = 0x2350; //unknown error + + public static final int ERROR_SUBSCRIBE_FAILED = 0x2351; // failed to create subscription + public static final int ERROR_UNSUPPORTED_ENCODING = 0x2352; // Unsupported encoding + + public static final int ERROR_JNI_TDENGINE_ERROR = 0x2353; // internal error of database + public static final int ERROR_JNI_CONNECTION_NULL = 0x2354; // JNI connection is NULL + public static final int ERROR_JNI_RESULT_SET_NULL = 0x2355; // invalid JNI result set + public static final int ERROR_JNI_NUM_OF_FIELDS_0 = 0x2356; // invalid num of fields + public static final int ERROR_JNI_SQL_NULL = 0x2357; // empty sql string + public static final int ERROR_JNI_FETCH_END = 0x2358; // fetch to the end of resultSet + public static final int ERROR_JNI_OUT_OF_MEMORY = 0x2359; // JNI alloc memory failed + + private static final HashSet errorNumbers; + + static { + errorNumbers = new HashSet(); + errorNumbers.add(ERROR_CONNECTION_CLOSED); + errorNumbers.add(ERROR_UNSUPPORTED_METHOD); + errorNumbers.add(ERROR_INVALID_VARIABLE); + errorNumbers.add(ERROR_STATEMENT_CLOSED); + errorNumbers.add(ERROR_RESULTSET_CLOSED); + errorNumbers.add(ERROR_INVALID_WITH_EXECUTEQUERY); + errorNumbers.add(ERROR_INVALID_WITH_EXECUTEUPDATE); + errorNumbers.add(ERROR_INVALID_FOR_EXECUTE_QUERY); + errorNumbers.add(ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE); + errorNumbers.add(ERROR_INVALID_FOR_EXECUTE_UPDATE); + errorNumbers.add(ERROR_INVALID_FOR_EXECUTE); + errorNumbers.add(ERROR_PARAMETER_INDEX_OUT_RANGE); + errorNumbers.add(ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); + errorNumbers.add(ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); + + /*****************************************************/ + errorNumbers.add(ERROR_SUBSCRIBE_FAILED); + errorNumbers.add(ERROR_UNSUPPORTED_ENCODING); + + errorNumbers.add(ERROR_JNI_TDENGINE_ERROR); + errorNumbers.add(ERROR_JNI_CONNECTION_NULL); + errorNumbers.add(ERROR_JNI_RESULT_SET_NULL); + errorNumbers.add(ERROR_JNI_NUM_OF_FIELDS_0); + errorNumbers.add(ERROR_JNI_SQL_NULL); + errorNumbers.add(ERROR_JNI_FETCH_END); + errorNumbers.add(ERROR_JNI_OUT_OF_MEMORY); + + } + + private TSDBErrorNumbers() { + } + + public static boolean contains(int errorNumber) { + return errorNumbers.contains(errorNumber); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index f918463439fba293171827001acf6930ab271b81..f3c4e025798e8e3fd8db0e5c1a1b25aa6fea3dc7 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -14,6 +14,8 @@ *****************************************************************************/ package com.taosdata.jdbc; +import com.taosdata.jdbc.utils.TaosInfo; + import java.sql.SQLException; import java.sql.SQLWarning; import java.util.List; @@ -21,6 +23,8 @@ import java.util.List; public class TSDBJNIConnector { private static volatile Boolean isInitialized = false; + private TaosInfo taosInfo = TaosInfo.getInstance(); + static { System.loadLibrary("taos"); System.out.println("java.library.path:" + System.getProperty("java.library.path")); @@ -34,7 +38,7 @@ public class TSDBJNIConnector { /** * Result set pointer for the current connection */ - private long taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; +// private long taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; /** * result set status in current connection @@ -91,15 +95,17 @@ public class TSDBJNIConnector { */ public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException { if (this.taos != TSDBConstants.JNI_NULL_POINTER) { - this.closeConnectionImp(this.taos); +// this.closeConnectionImp(this.taos); + closeConnection(); this.taos = TSDBConstants.JNI_NULL_POINTER; } this.taos = this.connectImp(host, port, dbName, user, password); if (this.taos == TSDBConstants.JNI_NULL_POINTER) { - throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(0L)), "", this.getErrCode(0l)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); } - + // invoke connectImp only here + taosInfo.conn_open_increment(); return true; } @@ -113,31 +119,43 @@ public class TSDBJNIConnector { public long executeQuery(String sql) throws SQLException { // close previous result set if the user forgets to invoke the // free method to close previous result set. - if (!this.isResultsetClosed) { - freeResultSet(taosResultSetPointer); - } +// if (!this.isResultsetClosed) { +// freeResultSet(taosResultSetPointer); +// } Long pSql = 0l; try { pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); + taosInfo.stmt_count_increment(); } catch (Exception e) { e.printStackTrace(); this.freeResultSetImp(this.taos, pSql); - throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding")); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_ENCODING); + } + if (pSql == TSDBConstants.JNI_CONNECTION_NULL) { + this.freeResultSetImp(this.taos, pSql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } + if (pSql == TSDBConstants.JNI_SQL_NULL) { + this.freeResultSetImp(this.taos, pSql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_SQL_NULL); + } + if (pSql == TSDBConstants.JNI_OUT_OF_MEMORY) { + this.freeResultSetImp(this.taos, pSql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_OUT_OF_MEMORY); } int code = this.getErrCode(pSql); - if (code != 0) { + if (code != TSDBConstants.JNI_SUCCESS) { affectedRows = -1; String msg = this.getErrMsg(pSql); - this.freeResultSetImp(this.taos, pSql); - throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code); + throw TSDBError.createSQLException(code, msg); } // Try retrieving result set for the executed SQL using the current connection pointer. - taosResultSetPointer = this.getResultSetImp(this.taos, pSql); - isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER); + pSql = this.getResultSetImp(this.taos, pSql); + isResultsetClosed = (pSql == TSDBConstants.JNI_NULL_POINTER); return pSql; } @@ -166,10 +184,9 @@ public class TSDBJNIConnector { * Get resultset pointer * Each connection should have a single open result set at a time */ - public long getResultSet() { - return taosResultSetPointer; - } - +// public long getResultSet() { +// return taosResultSetPointer; +// } private native long getResultSetImp(long connection, long pSql); public boolean isUpdateQuery(long pSql) { @@ -181,16 +198,16 @@ public class TSDBJNIConnector { /** * Free resultset operation from C to release resultset pointer by JNI */ - public int freeResultSet(long result) { + public int freeResultSet(long pSql) { int res = TSDBConstants.JNI_SUCCESS; - if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - throw new RuntimeException("Invalid result set pointer"); - } +// if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { +// throw new RuntimeException("Invalid result set pointer"); +// } - if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, result); - taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - } +// if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { + res = this.freeResultSetImp(this.taos, pSql); +// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; +// } isResultsetClosed = true; return res; @@ -200,16 +217,15 @@ public class TSDBJNIConnector { * Close the open result set which is associated to the current connection. If the result set is already * closed, return 0 for success. */ - public int freeResultSet() { - int resCode = TSDBConstants.JNI_SUCCESS; - if (!isResultsetClosed) { - resCode = this.freeResultSetImp(this.taos, this.taosResultSetPointer); - taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - isResultsetClosed = true; - } - return resCode; - } - +// public int freeResultSet() { +// int resCode = TSDBConstants.JNI_SUCCESS; +// if (!isResultsetClosed) { +// resCode = this.freeResultSetImp(this.taos, this.taosResultSetPointer); +// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; +// isResultsetClosed = true; +// } +// return resCode; +// } private native int freeResultSetImp(long connection, long result); /** @@ -244,10 +260,11 @@ public class TSDBJNIConnector { private native int fetchRowImp(long connection, long resultSet, TSDBResultSetRowData rowData); public int fetchBlock(long resultSet, TSDBResultSetBlockData blockData) { - return this.fetchBlockImp(this.taos, resultSet, blockData); + return this.fetchBlockImp(this.taos, resultSet, blockData); } - + private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData); + /** * Execute close operation from C to release connection pointer by JNI * @@ -256,12 +273,14 @@ public class TSDBJNIConnector { public void closeConnection() throws SQLException { int code = this.closeConnectionImp(this.taos); if (code < 0) { - throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(0l)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); } else if (code == 0) { this.taos = TSDBConstants.JNI_NULL_POINTER; } else { throw new SQLException("Undefined error code returned by TDengine when closing a connection"); } + // invoke closeConnectionImpl only here + taosInfo.connect_close_increment(); } private native int closeConnectionImp(long connection); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java deleted file mode 100644 index d9227523d4ac623d23d85be6376a1530ca606751..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java +++ /dev/null @@ -1,75 +0,0 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; - -import java.sql.ParameterMetaData; -import java.sql.SQLException; - -public class TSDBParameterMetaData implements ParameterMetaData { - @Override - public int getParameterCount() throws SQLException { - return 0; - } - - @Override - public int isNullable(int param) throws SQLException { - return 0; - } - - @Override - public boolean isSigned(int param) throws SQLException { - return false; - } - - @Override - public int getPrecision(int param) throws SQLException { - return 0; - } - - @Override - public int getScale(int param) throws SQLException { - return 0; - } - - @Override - public int getParameterType(int param) throws SQLException { - return 0; - } - - @Override - public String getParameterTypeName(int param) throws SQLException { - return null; - } - - @Override - public String getParameterClassName(int param) throws SQLException { - return null; - } - - @Override - public int getParameterMode(int param) throws SQLException { - return 0; - } - - @Override - public T unwrap(Class iface) throws SQLException { - return null; - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return false; - } -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 230943fd53ae8fa36b9de40f851a5c263c052f0d..d294c8974543db409187a60cf77ea03c98f5ff17 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -32,7 +32,7 @@ import java.util.regex.Pattern; public class TSDBPreparedStatement extends TSDBStatement implements PreparedStatement { protected String rawSql; protected String sql; - protected ArrayList parameters = new ArrayList(); + protected ArrayList parameters = new ArrayList<>(); //start with insert or import and is case-insensitive private static Pattern savePattern = Pattern.compile("(?i)^\\s*(insert|import)"); @@ -41,6 +41,7 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat private boolean isSaved; private SavedPreparedStatement savedPreparedStatement; + private ParameterMetaData parameterMetaData; TSDBPreparedStatement(TSDBConnection connection, TSDBJNIConnector connecter, String sql) { super(connection, connecter); @@ -53,7 +54,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat this.isSaved = isSavedSql(this.rawSql); if (this.isSaved) { - try { this.savedPreparedStatement = new SavedPreparedStatement(this.rawSql, this); } catch (SQLException e) { @@ -90,17 +90,12 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat this.parameters = parameters; } - public String getRawSql() { - return rawSql; - } - /* * Some of the SQLs sent by other popular frameworks or tools like Spark, contains syntax that cannot be parsed by * the TDengine client. Thus, some simple parsers/filters are intentionally added in this JDBC implementation in * order to process those supported SQLs. */ private void preprocessSql() { - /***** For processing some of Spark SQLs*****/ // should replace it first this.rawSql = this.rawSql.replaceAll("or (.*) is null", ""); @@ -149,7 +144,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat rawSql = rawSql.replace(matcher.group(1), tableFullName); } /***** for inner queries *****/ - } /** @@ -194,99 +188,191 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat } } + private boolean isSupportedSQLType(int sqlType) { + switch (sqlType) { + case Types.TIMESTAMP: + case Types.INTEGER: + case Types.BIGINT: + case Types.FLOAT: + case Types.DOUBLE: + case Types.SMALLINT: + case Types.TINYINT: + case Types.BOOLEAN: + case Types.BINARY: + case Types.NCHAR: + return true; + case Types.ARRAY: + case Types.BIT: + case Types.BLOB: + case Types.CHAR: + case Types.CLOB: + case Types.DATALINK: + case Types.DATE: + case Types.DECIMAL: + case Types.DISTINCT: + case Types.JAVA_OBJECT: + case Types.LONGNVARCHAR: + case Types.LONGVARBINARY: + case Types.LONGVARCHAR: + case Types.NCLOB: + case Types.NULL: + case Types.NUMERIC: + case Types.NVARCHAR: + case Types.OTHER: + case Types.REAL: + case Types.REF: + case Types.REF_CURSOR: + case Types.ROWID: + case Types.SQLXML: + case Types.STRUCT: + case Types.TIME: + case Types.TIME_WITH_TIMEZONE: + case Types.TIMESTAMP_WITH_TIMEZONE: + case Types.VARBINARY: + case Types.VARCHAR: + default: + return false; + } + } + @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { - setObject(parameterIndex, new String("NULL")); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isSupportedSQLType(sqlType) || parameterIndex < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); +// if (parameterIndex >= parameters.size()) +// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_BOUNDARY); + + setObject(parameterIndex, "NULL"); } @Override public void setBoolean(int parameterIndex, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, x); } @Override public void setByte(int parameterIndex, byte x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setShort(int parameterIndex, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setInt(int parameterIndex, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setLong(int parameterIndex, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setFloat(int parameterIndex, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setDouble(int parameterIndex, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setString(int parameterIndex, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setBytes(int parameterIndex, byte[] x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setDate(int parameterIndex, Date x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTime(int parameterIndex, Time x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void clearParameters() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); parameters.clear(); } @Override public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -313,9 +399,8 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat if (isSaved) { this.savedPreparedStatement.addBatch(); } else { - if (this.batchedArgs == null) { - batchedArgs = new ArrayList(); + batchedArgs = new ArrayList<>(); } super.addBatch(getNativeSql()); } @@ -323,156 +408,226 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setRef(int parameterIndex, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBlob(int parameterIndex, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setClob(int parameterIndex, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setArray(int parameterIndex, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public ResultSetMetaData getMetaData() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); +// return this.getResultSet().getMetaData(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // TODO： + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setURL(int parameterIndex, URL x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public ParameterMetaData getParameterMetaData() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + //TODO: parameterMetaData not supported +// return null; + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setRowId(int parameterIndex, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNString(int parameterIndex, String value) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNClob(int parameterIndex, NClob value) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + //TODO: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setClob(int parameterIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } @Override public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNClob(int parameterIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index 84a3f58f4692a99737af7d93c5578fc7a5a09c27..80ff49253016c632b6bf85f4756fe2d702a9bffc 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -14,1086 +14,429 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.io.InputStream; -import java.io.Reader; import java.math.BigDecimal; -import java.net.URL; -import java.sql.Array; -import java.sql.Blob; -import java.sql.Clob; -import java.sql.Date; -import java.sql.NClob; -import java.sql.Ref; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.RowId; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; +import java.sql.*; import java.util.ArrayList; -import java.util.Calendar; -import java.util.Iterator; import java.util.List; -import java.util.Map; - -@SuppressWarnings("unused") -public class TSDBResultSet implements ResultSet { - private TSDBJNIConnector jniConnector = null; - - private long resultSetPointer = 0L; - private List columnMetaDataList = new ArrayList(); - - private TSDBResultSetRowData rowData; - private TSDBResultSetBlockData blockData; - - private boolean batchFetch = false; - private boolean lastWasNull = false; - private final int COLUMN_INDEX_START_VALUE = 1; - - private int rowIndex = 0; - - public TSDBJNIConnector getJniConnector() { - return jniConnector; - } - - public void setJniConnector(TSDBJNIConnector jniConnector) { - this.jniConnector = jniConnector; - } - - public long getResultSetPointer() { - return resultSetPointer; - } - - public void setResultSetPointer(long resultSetPointer) { - this.resultSetPointer = resultSetPointer; - } - - public void setBatchFetch(boolean batchFetch) { - this.batchFetch = batchFetch; - } - - public Boolean getBatchFetch() { - return this.batchFetch; - } - - public List getColumnMetaDataList() { - return columnMetaDataList; - } - - public void setColumnMetaDataList(List columnMetaDataList) { - this.columnMetaDataList = columnMetaDataList; - } - - public TSDBResultSetRowData getRowData() { - return rowData; - } - - public void setRowData(TSDBResultSetRowData rowData) { - this.rowData = rowData; - } - - public boolean isLastWasNull() { - return lastWasNull; - } - - public void setLastWasNull(boolean lastWasNull) { - this.lastWasNull = lastWasNull; - } - - public TSDBResultSet() { - } - - public TSDBResultSet(TSDBJNIConnector connector, long resultSetPointer) throws SQLException { - this.jniConnector = connector; - this.resultSetPointer = resultSetPointer; - int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } - - this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size()); - this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size()); - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean next() throws SQLException { - if (this.getBatchFetch()) { - if (this.blockData.forward()) { - return true; - } - - int code = this.jniConnector.fetchBlock(this.resultSetPointer, this.blockData); - this.blockData.reset(); - - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } else if (code == TSDBConstants.JNI_FETCH_END) { - return false; - } - - return true; - } else { - if (rowData != null) { - this.rowData.clear(); - } - - int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } else if (code == TSDBConstants.JNI_FETCH_END) { - return false; - } else { - return true; - } - } - } - - public void close() throws SQLException { - if (this.jniConnector != null) { - int code = this.jniConnector.freeResultSet(this.resultSetPointer); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } - } - } - - public boolean wasNull() throws SQLException { - return this.lastWasNull; - } - - public String getString(int columnIndex) throws SQLException { - String res = null; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getString(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getString(colIndex); - } - } - - public boolean getBoolean(int columnIndex) throws SQLException { - boolean res = false; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getBoolean(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - } else { - return this.blockData.getBoolean(colIndex); - } - - return res; - } - - public byte getByte(int columnIndex) throws SQLException { - byte res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = (byte) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (byte) this.blockData.getInt(colIndex); - } - } - - public short getShort(int columnIndex) throws SQLException { - short res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = (short) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (short) this.blockData.getInt(colIndex); - } - } - - public int getInt(int columnIndex) throws SQLException { - int res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getInt(colIndex); - } - - } - - public long getLong(int columnIndex) throws SQLException { - long res = 0l; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getLong(colIndex); - } - } - - public float getFloat(int columnIndex) throws SQLException { - float res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getFloat(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (float) this.blockData.getDouble(colIndex); - } - } - - public double getDouble(int columnIndex) throws SQLException { - double res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getDouble(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getDouble(colIndex); - } - } - - /* - * (non-Javadoc) - * - * @see java.sql.ResultSet#getBigDecimal(int, int) - * - * @deprecated Use {@code getBigDecimal(int columnIndex)} or {@code - * getBigDecimal(String columnLabel)} - */ - @Deprecated - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - return new BigDecimal(getLong(columnIndex)); - } - - public byte[] getBytes(int columnIndex) throws SQLException { - return getString(columnIndex).getBytes(); - } - - public Date getDate(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Time getTime(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Timestamp getTimestamp(int columnIndex) throws SQLException { - Timestamp res = null; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getTimestamp(colIndex); - } - return res; - } else { - return this.blockData.getTimestamp(columnIndex); - } - } - - public InputStream getAsciiStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * (non-Javadoc) - * - * @see java.sql.ResultSet#getUnicodeStream(int) - * - * * @deprecated use getCharacterStream in place of - * getUnicodeStream - */ - @Deprecated - public InputStream getUnicodeStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public InputStream getBinaryStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getString(String columnLabel) throws SQLException { - return this.getString(this.findColumn(columnLabel)); - } - - public boolean getBoolean(String columnLabel) throws SQLException { - return this.getBoolean(this.findColumn(columnLabel)); - } - - public byte getByte(String columnLabel) throws SQLException { - return this.getByte(this.findColumn(columnLabel)); - } - - public short getShort(String columnLabel) throws SQLException { - return this.getShort(this.findColumn(columnLabel)); - } - - public int getInt(String columnLabel) throws SQLException { - return this.getInt(this.findColumn(columnLabel)); - } - - public long getLong(String columnLabel) throws SQLException { - return this.getLong(this.findColumn(columnLabel)); - } - - public float getFloat(String columnLabel) throws SQLException { - return this.getFloat(this.findColumn(columnLabel)); - } - - public double getDouble(String columnLabel) throws SQLException { - return this.getDouble(this.findColumn(columnLabel)); - } - - /* - * used by spark - */ - @Deprecated - public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { - return this.getBigDecimal(this.findColumn(columnLabel), scale); - } - - public byte[] getBytes(String columnLabel) throws SQLException { - return this.getBytes(this.findColumn(columnLabel)); - } - - public Date getDate(String columnLabel) throws SQLException { - return this.getDate(this.findColumn(columnLabel)); - } - - public Time getTime(String columnLabel) throws SQLException { - return this.getTime(this.findColumn(columnLabel)); - } - - public Timestamp getTimestamp(String columnLabel) throws SQLException { - return this.getTimestamp(this.findColumn(columnLabel)); - } - - public InputStream getAsciiStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Deprecated - public InputStream getUnicodeStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public InputStream getBinaryStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getCursorName() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSetMetaData getMetaData() throws SQLException { - return new TSDBResultSetMetaData(this.columnMetaDataList); - } - - public Object getObject(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - return this.rowData.get(colIndex); - } else { - return this.blockData.get(colIndex); - } - } - - public Object getObject(String columnLabel) throws SQLException { - return this.getObject(this.findColumn(columnLabel)); - } - - public int findColumn(String columnLabel) throws SQLException { - Iterator colMetaDataIt = this.columnMetaDataList.iterator(); - while (colMetaDataIt.hasNext()) { - ColumnMetaData colMetaData = colMetaDataIt.next(); - if (colMetaData.getColName() != null && colMetaData.getColName().equalsIgnoreCase(columnLabel)) { - return colMetaData.getColIndex() + 1; - } - } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - public Reader getCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Reader getCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * used by spark - */ - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - return new BigDecimal(this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType())); - } else { - return new BigDecimal(this.blockData.getLong(colIndex)); - } - } - - public BigDecimal getBigDecimal(String columnLabel) throws SQLException { - return this.getBigDecimal(this.findColumn(columnLabel)); - } - - public boolean isBeforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isAfterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void beforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void afterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean first() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean last() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean absolute(int row) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean relative(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean previous() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchDirection(int direction) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchDirection() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchSize(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchSize() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getType() throws SQLException { - return ResultSet.TYPE_FORWARD_ONLY; - } - - public int getConcurrency() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowUpdated() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowInserted() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowDeleted() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNull(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - public void updateBoolean(int columnIndex, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateByte(int columnIndex, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateShort(int columnIndex, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateInt(int columnIndex, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateLong(int columnIndex, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateFloat(int columnIndex, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDouble(int columnIndex, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateString(int columnIndex, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBytes(int columnIndex, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDate(int columnIndex, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTime(int columnIndex, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(int columnIndex, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNull(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBoolean(String columnLabel, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateByte(String columnLabel, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateShort(String columnLabel, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateInt(String columnLabel, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateLong(String columnLabel, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateFloat(String columnLabel, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDouble(String columnLabel, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateString(String columnLabel, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBytes(String columnLabel, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDate(String columnLabel, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTime(String columnLabel, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(String columnLabel, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void insertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void deleteRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void refreshRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void cancelRowUpdates() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void moveToInsertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void moveToCurrentRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Statement getStatement() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Object getObject(int columnIndex, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Ref getRef(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob getBlob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob getClob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Array getArray(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Object getObject(String columnLabel, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Ref getRef(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob getBlob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob getClob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Array getArray(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Date getDate(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Date getDate(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Time getTime(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Time getTime(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public URL getURL(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public URL getURL(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRef(int columnIndex, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRef(String columnLabel, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateArray(int columnIndex, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateArray(String columnLabel, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public RowId getRowId(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public RowId getRowId(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRowId(int columnIndex, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRowId(String columnLabel, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getHoldability() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isClosed() throws SQLException { - boolean isClosed = true; - if (jniConnector != null) { - isClosed = jniConnector.isResultsetClosed(); - } - return isClosed; - } - - public void updateNString(int columnIndex, String nString) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNString(String columnLabel, String nString) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, NClob nClob) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, NClob nClob) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob getNClob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob getNClob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML getSQLXML(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML getSQLXML(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getNString(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - return (String) rowData.get(colIndex); - } - - public String getNString(String columnLabel) throws SQLException { - return (String) this.getString(columnLabel); - } - - public Reader getNCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Reader getNCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - private int getTrueColumnIndex(int columnIndex) throws SQLException { - if (columnIndex < this.COLUMN_INDEX_START_VALUE) { - throw new SQLException("Column Index out of range, " + columnIndex + " < " + this.COLUMN_INDEX_START_VALUE); - } - - int numOfCols = this.columnMetaDataList.size(); - if (columnIndex > numOfCols) { - throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols); - } - - return columnIndex - 1; - } +public class TSDBResultSet extends AbstractResultSet implements ResultSet { + private TSDBJNIConnector jniConnector; + + private final TSDBStatement statement; + private long resultSetPointer = 0L; + private List columnMetaDataList = new ArrayList<>(); + + private TSDBResultSetRowData rowData; + private TSDBResultSetBlockData blockData; + + private boolean batchFetch = false; + private boolean lastWasNull = false; + private final int COLUMN_INDEX_START_VALUE = 1; + + public void setBatchFetch(boolean batchFetch) { + this.batchFetch = batchFetch; + } + + public Boolean getBatchFetch() { + return this.batchFetch; + } + + public void setColumnMetaDataList(List columnMetaDataList) { + this.columnMetaDataList = columnMetaDataList; + } + + public TSDBResultSetRowData getRowData() { + return rowData; + } + + public TSDBResultSet(TSDBStatement statement, TSDBJNIConnector connector, long resultSetPointer) throws SQLException { + this.statement = statement; + this.jniConnector = connector; + this.resultSetPointer = resultSetPointer; + + int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } + if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); + } + this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size()); + this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size()); + } + + public boolean next() throws SQLException { + if (this.getBatchFetch()) { + if (this.blockData.forward()) { + return true; + } + + int code = this.jniConnector.fetchBlock(this.resultSetPointer, this.blockData); + this.blockData.reset(); + + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); + } else if (code == TSDBConstants.JNI_FETCH_END) { + return false; + } + + return true; + } else { + if (rowData != null) { + this.rowData.clear(); + } + + int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); + } else if (code == TSDBConstants.JNI_FETCH_END) { + return false; + } else { + return true; + } + } + } + + public void close() throws SQLException { + if (this.jniConnector != null) { + int code = this.jniConnector.freeResultSet(this.resultSetPointer); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); + } + } + } + + public boolean wasNull() throws SQLException { + return this.lastWasNull; + } + + public String getString(int columnIndex) throws SQLException { + String res = null; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getString(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getString(colIndex); + } + } + + public boolean getBoolean(int columnIndex) throws SQLException { + boolean res = false; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getBoolean(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + } else { + return this.blockData.getBoolean(colIndex); + } + + return res; + } + + public byte getByte(int columnIndex) throws SQLException { + byte res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = (byte) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return (byte) this.blockData.getInt(colIndex); + } + } + + public short getShort(int columnIndex) throws SQLException { + short res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = (short) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return (short) this.blockData.getInt(colIndex); + } + } + + public int getInt(int columnIndex) throws SQLException { + int res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getInt(colIndex); + } + + } + + public long getLong(int columnIndex) throws SQLException { + long res = 0L; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getLong(colIndex); + } + } + + public float getFloat(int columnIndex) throws SQLException { + float res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getFloat(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return (float) this.blockData.getDouble(colIndex); + } + } + + public double getDouble(int columnIndex) throws SQLException { + double res = 0; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getDouble(colIndex, this.columnMetaDataList.get(colIndex).getColType()); + } + return res; + } else { + return this.blockData.getDouble(colIndex); + } + } + + @Deprecated + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + return new BigDecimal(getLong(columnIndex)); + } + + public byte[] getBytes(int columnIndex) throws SQLException { + return getString(columnIndex).getBytes(); + } + + public Timestamp getTimestamp(int columnIndex) throws SQLException { + Timestamp res = null; + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + if (!lastWasNull) { + res = this.rowData.getTimestamp(colIndex); + } + return res; + } else { + return this.blockData.getTimestamp(columnIndex); + } + } + + public ResultSetMetaData getMetaData() throws SQLException { + return new TSDBResultSetMetaData(this.columnMetaDataList); + } + + @Override + public Object getObject(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + return this.rowData.get(colIndex); + } else { + return this.blockData.get(colIndex); + } + } + + @Override + public Object getObject(String columnLabel) throws SQLException { + return this.getObject(this.findColumn(columnLabel)); + } + + public int findColumn(String columnLabel) throws SQLException { + for (ColumnMetaData colMetaData : this.columnMetaDataList) { + if (colMetaData.getColName() != null && colMetaData.getColName().equalsIgnoreCase(columnLabel)) { + return colMetaData.getColIndex() + 1; + } + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + + if (!this.getBatchFetch()) { + this.lastWasNull = this.rowData.wasNull(colIndex); + return new BigDecimal(this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType())); + } else { + return new BigDecimal(this.blockData.getLong(colIndex)); + } + } + + @Override + public boolean isBeforeFirst() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isAfterLast() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isFirst() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isLast() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void beforeFirst() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void afterLast() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean first() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean last() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean absolute(int row) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean relative(int rows) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean previous() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public Statement getStatement() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + return this.statement; + } + + public boolean isClosed() throws SQLException { + //TODO: check if need release resources + boolean isClosed = true; + if (jniConnector != null) { + isClosed = jniConnector.isResultsetClosed(); + } + return isClosed; + } + + public String getNString(int columnIndex) throws SQLException { + int colIndex = getTrueColumnIndex(columnIndex); + return (String) rowData.get(colIndex); + } + + private int getTrueColumnIndex(int columnIndex) throws SQLException { + if (columnIndex < this.COLUMN_INDEX_START_VALUE) { + throw new SQLException("Column Index out of range, " + columnIndex + " < " + this.COLUMN_INDEX_START_VALUE); + } + + int numOfCols = this.columnMetaDataList.size(); + if (columnIndex > numOfCols) { + throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols); + } + return columnIndex - 1; + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java index d6d69bd8b02db7de9350c023e696f4cf2e2f5e65..e0b1c246cbea246f885ab4c56c7ece0be212ff66 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java @@ -20,7 +20,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.List; -public class TSDBResultSetMetaData implements ResultSetMetaData { +public class TSDBResultSetMetaData extends WrapperImpl implements ResultSetMetaData { List colMetaDataList = null; @@ -28,14 +28,6 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { this.colMetaDataList = metaDataList; } - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - public int getColumnCount() throws SQLException { return colMetaDataList.size(); } @@ -94,7 +86,7 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { } public String getSchemaName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public int getPrecision(int column) throws SQLException { @@ -125,18 +117,18 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { } public String getTableName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public String getCatalogName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public int getColumnType(int column) throws SQLException { ColumnMetaData meta = this.colMetaDataList.get(column - 1); switch (meta.getColType()) { case TSDBConstants.TSDB_DATA_TYPE_BOOL: - return java.sql.Types.BIT; + return Types.BOOLEAN; case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return java.sql.Types.TINYINT; case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: @@ -150,13 +142,13 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return java.sql.Types.DOUBLE; case TSDBConstants.TSDB_DATA_TYPE_BINARY: - return java.sql.Types.CHAR; + return Types.BINARY; case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - return java.sql.Types.BIGINT; + return java.sql.Types.TIMESTAMP; case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - return java.sql.Types.CHAR; + return Types.NCHAR; } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } public String getColumnTypeName(int column) throws SQLException { @@ -173,7 +165,7 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { } public boolean isDefinitelyWritable(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public String getColumnClassName(int column) throws SQLException { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index c57f19550dd14719baecb835d76263df1e6a669b..6518bf10e444a05073206e1ef72b8f21a87e26b1 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -26,9 +26,9 @@ public class TSDBResultSetRowData { public TSDBResultSetRowData(int colSize) { this.setColSize(colSize); } - + public TSDBResultSetRowData() { - this.data = new ArrayList(); + this.data = new ArrayList<>(); this.setColSize(0); } @@ -39,7 +39,7 @@ public class TSDBResultSetRowData { if (this.colSize == 0) { return; } - this.data = new ArrayList(colSize); + this.data = new ArrayList<>(colSize); this.data.addAll(Collections.nCopies(this.colSize, null)); } @@ -53,7 +53,7 @@ public class TSDBResultSetRowData { public boolean getBoolean(int col, int srcType) throws SQLException { Object obj = data.get(col); - + switch(srcType) { case TSDBConstants.TSDB_DATA_TYPE_BOOL: return (Boolean) obj; case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj) == 1.0? Boolean.TRUE:Boolean.FALSE; @@ -64,10 +64,10 @@ public class TSDBResultSetRowData { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return ((Long) obj) == 1L? Boolean.TRUE:Boolean.FALSE; } - + return Boolean.TRUE; } - + public void setByte(int col, byte value) { data.set(col, value); } @@ -82,20 +82,20 @@ public class TSDBResultSetRowData { public int getInt(int col, int srcType) throws SQLException { Object obj = data.get(col); - + switch(srcType) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj).intValue(); - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return ((Double)obj).intValue(); - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return (Byte) obj; - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT:return (Short) obj; - case TSDBConstants.TSDB_DATA_TYPE_INT: return (Integer) obj; - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return ((Long) obj).intValue(); - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Integer.parseInt((String) obj); + case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj).intValue(); + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return ((Double)obj).intValue(); + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return (Byte) obj; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT:return (Short) obj; + case TSDBConstants.TSDB_DATA_TYPE_INT: return (Integer) obj; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return ((Long) obj).intValue(); + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Integer.parseInt((String) obj); } - + return 0; } @@ -105,7 +105,7 @@ public class TSDBResultSetRowData { public long getLong(int col, int srcType) throws SQLException { Object obj = data.get(col); - + switch(srcType) { case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj).longValue(); @@ -116,9 +116,9 @@ public class TSDBResultSetRowData { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return (Long) obj; case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Long.parseLong((String) obj); + case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Long.parseLong((String) obj); } - + return 0; } @@ -128,7 +128,7 @@ public class TSDBResultSetRowData { public float getFloat(int col, int srcType) throws SQLException { Object obj = data.get(col); - + switch(srcType) { case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return (Float) obj; @@ -139,7 +139,7 @@ public class TSDBResultSetRowData { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return (Long) obj; } - + return 0; } @@ -149,7 +149,7 @@ public class TSDBResultSetRowData { public double getDouble(int col, int srcType) throws SQLException { Object obj = data.get(col); - + switch(srcType) { case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return (Float) obj; @@ -160,7 +160,7 @@ public class TSDBResultSetRowData { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return (Long) obj; } - + return 0; } @@ -180,7 +180,7 @@ public class TSDBResultSetRowData { * The original type may not be a string type, but will be converted to by calling this method * @param col column index * @return - * @throws SQLException + * @throws SQLException */ public String getString(int col, int srcType) throws SQLException { if (srcType == TSDBConstants.TSDB_DATA_TYPE_BINARY || srcType == TSDBConstants.TSDB_DATA_TYPE_NCHAR) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java index 059962a7a120073c17258652f9fb3609cc5072be..48854e773f89a45784de3cd709ec5bbe6185e09b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java @@ -1153,11 +1153,11 @@ public class TSDBResultSetWrapper implements ResultSet { } public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index cdd88b825e2a8f6ba81c131bfd1214ffbe46f32c..996ea4a185d97b2294ac06cacccd08862de2de94 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -14,32 +14,20 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; -public class TSDBStatement implements Statement { - private TSDBJNIConnector connector = null; - - /** - * To store batched commands - */ - protected List batchedArgs; - - /** - * Timeout for a query - */ - protected int queryTimeout = 0; - - private Long pSql = 0l; +public class TSDBStatement extends AbstractStatement { + private TSDBJNIConnector connector; /** * Status of current statement */ - private boolean isClosed = true; - private int affectedRows = 0; - + private boolean isClosed; + private int affectedRows = -1; private TSDBConnection connection; + private TSDBResultSet resultSet; public void setConnection(TSDBConnection connection) { this.connection = connection; @@ -48,285 +36,96 @@ public class TSDBStatement implements Statement { TSDBStatement(TSDBConnection connection, TSDBJNIConnector connector) { this.connection = connection; this.connector = connector; - this.isClosed = false; - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public ResultSet executeQuery(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - - // TODO make sure it is not a update query - pSql = this.connector.executeQuery(sql); - - long resultSetPointer = this.connector.getResultSet(); - - if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { - this.connector.freeResultSet(pSql); - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - - // create/insert/update/delete/alter - if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { - this.connector.freeResultSet(pSql); - return null; - } - - if (!this.connector.isUpdateQuery(pSql)) { - TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer); - res.setBatchFetch(this.connection.getBatchFetch()); - return res; - } else { + // check if closed + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + //TODO: 如果在executeQuery方法中执行insert语句，那么先执行了SQL，再通过pSql来检查是否为一个insert语句，但这个insert SQL已经执行成功了 + + // execute query + long pSql = this.connector.executeQuery(sql); + // if pSql is create/insert/update/delete/alter SQL + if (this.connector.isUpdateQuery(pSql)) { this.connector.freeResultSet(pSql); - return null; + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY); } + TSDBResultSet res = new TSDBResultSet(this, this.connector, pSql); + res.setBatchFetch(this.connection.getBatchFetch()); + return res; } public int executeUpdate(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - // TODO check if current query is update query - pSql = this.connector.executeQuery(sql); - long resultSetPointer = this.connector.getResultSet(); - - if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { + long pSql = this.connector.executeQuery(sql); + // if pSql is create/insert/update/delete/alter SQL + if (!this.connector.isUpdateQuery(pSql)) { this.connector.freeResultSet(pSql); - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEUPDATE); } - - this.affectedRows = this.connector.getAffectedRows(pSql); + int affectedRows = this.connector.getAffectedRows(pSql); this.connector.freeResultSet(pSql); - - return this.affectedRows; - } - - public String getErrorMsg(long pSql) { - return this.connector.getErrMsg(pSql); + return affectedRows; } public void close() throws SQLException { if (!isClosed) { - if (!this.connector.isResultsetClosed()) { - this.connector.freeResultSet(); - } + if (this.resultSet != null) + this.resultSet.close(); isClosed = true; } } - public int getMaxFieldSize() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setMaxFieldSize(int max) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getMaxRows() throws SQLException { - // always set maxRows to zero, meaning unlimitted rows in a resultSet - return 0; - } - - public void setMaxRows(int max) throws SQLException { - // always set maxRows to zero, meaning unlimited rows in a resultSet - } - - public void setEscapeProcessing(boolean enable) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getQueryTimeout() throws SQLException { - return queryTimeout; - } - - public void setQueryTimeout(int seconds) throws SQLException { - this.queryTimeout = seconds; - } - - public void cancel() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setCursorName(String name) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - public boolean execute(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - boolean res = true; - pSql = this.connector.executeQuery(sql); - long resultSetPointer = this.connector.getResultSet(); - - if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { + // check if closed + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // execute query + long pSql = this.connector.executeQuery(sql); + // if pSql is create/insert/update/delete/alter SQL + if (this.connector.isUpdateQuery(pSql)) { + this.affectedRows = this.connector.getAffectedRows(pSql); this.connector.freeResultSet(pSql); - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { - // no result set is retrieved - this.connector.freeResultSet(pSql); - res = false; + return false; } - return res; + this.resultSet = new TSDBResultSet(this, this.connector, pSql); + this.resultSet.setBatchFetch(this.connection.getBatchFetch()); + return true; } public ResultSet getResultSet() throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - long resultSetPointer = connector.getResultSet(); - TSDBResultSet resSet = null; - if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - resSet = new TSDBResultSet(connector, resultSetPointer); - } - return resSet; + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); +// long resultSetPointer = connector.getResultSet(); +// TSDBResultSet resSet = null; +// if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { +// resSet = new TSDBResultSet(connector, resultSetPointer); +// } + return this.resultSet; } public int getUpdateCount() throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); return this.affectedRows; } - public boolean getMoreResults() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchDirection(int direction) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchDirection() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * used by spark - */ - public void setFetchSize(int rows) throws SQLException { - } - - /* - * used by spark - */ - public int getFetchSize() throws SQLException { - return 4096; - } - - public int getResultSetConcurrency() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getResultSetType() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void addBatch(String sql) throws SQLException { - if (batchedArgs == null) { - batchedArgs = new ArrayList<>(); - } - batchedArgs.add(sql); - } - - public void clearBatch() throws SQLException { - batchedArgs.clear(); - } - - public int[] executeBatch() throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - if (batchedArgs == null) { - throw new SQLException(TSDBConstants.WrapErrMsg("Batch is empty!")); - } else { - int[] res = new int[batchedArgs.size()]; - for (int i = 0; i < batchedArgs.size(); i++) { - res[i] = executeUpdate(batchedArgs.get(i)); - } - return res; - } - } - public Connection getConnection() throws SQLException { - if (this.connector != null) - return this.connection; - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean getMoreResults(int current) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSet getGeneratedKeys() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int executeUpdate(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean execute(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean execute(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getResultSetHoldability() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (this.connector == null) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + return this.connection; } public boolean isClosed() throws SQLException { return isClosed; } - public void setPoolable(boolean poolable) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isPoolable() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - public void closeOnCompletion() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isCloseOnCompletion() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java index deffd9aa2ae88802f71af5cbec66c5896cf4e19a..dd0d0d5b7b2744e0e6d28a3d73ec2b29fab37743 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java @@ -14,15 +14,11 @@ *****************************************************************************/ package com.taosdata.jdbc; -import javax.management.OperationsException; import java.sql.SQLException; -import java.util.Map; -import java.util.TimerTask; -import java.util.concurrent.*; public class TSDBSubscribe { - private TSDBJNIConnector connecter = null; - private long id = 0; + private final TSDBJNIConnector connecter; + private final long id; TSDBSubscribe(TSDBJNIConnector connecter, long id) throws SQLException { if (null != connecter) { @@ -36,9 +32,8 @@ public class TSDBSubscribe { /** * consume * - * @throws OperationsException, SQLException */ - public TSDBResultSet consume() throws OperationsException, SQLException { + public TSDBResultSet consume() throws SQLException { if (this.connecter.isClosed()) { throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); } @@ -50,7 +45,7 @@ public class TSDBSubscribe { } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { return null; } else { - return new TSDBResultSet(this.connecter, resultSetPointer); + return new TSDBResultSet(null, this.connecter, resultSetPointer); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..5b7539d434e0c5595d75da0a50baca9ab59c953b --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java @@ -0,0 +1,21 @@ +package com.taosdata.jdbc; + +import java.sql.SQLException; +import java.sql.Wrapper; + +public class WrapperImpl implements Wrapper { + + @Override + public T unwrap(Class iface) throws SQLException { + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return iface.isInstance(this); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index 6b0937a9b7c50c25d03459d18e01b807b5c00c3c..0484701f68f317dab2a1415b321d093d488c5b6a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -1,315 +1,64 @@ package com.taosdata.jdbc.rs; -import com.taosdata.jdbc.TSDBConstants; +import com.taosdata.jdbc.*; import java.sql.*; -import java.util.Map; import java.util.Properties; -import java.util.concurrent.Executor; -public class RestfulConnection implements Connection { +public class RestfulConnection extends AbstractConnection { private final String host; private final int port; - private final Properties props; - private final String database; private final String url; + private volatile String database; + /******************************************************/ + private boolean isClosed; + private final DatabaseMetaData metadata; public RestfulConnection(String host, String port, Properties props, String database, String url) { this.host = host; this.port = Integer.parseInt(port); - this.props = props; this.database = database; this.url = url; + this.metadata = new RestfulDatabaseMetaData(url, props.getProperty(TSDBDriver.PROPERTY_KEY_USER), this); } @Override public Statement createStatement() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg("restful TDengine connection is closed.")); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + ; + return new RestfulStatement(this, database); } @Override public PreparedStatement prepareStatement(String sql) throws SQLException { - return null; - } - - @Override - public CallableStatement prepareCall(String sql) throws SQLException { - return null; - } - - @Override - public String nativeSQL(String sql) throws SQLException { - return null; - } - - @Override - public void setAutoCommit(boolean autoCommit) throws SQLException { - - } - - @Override - public boolean getAutoCommit() throws SQLException { - return false; - } - - @Override - public void commit() throws SQLException { - - } - - @Override - public void rollback() throws SQLException { - + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return new RestfulPreparedStatement(this, database, sql); } @Override public void close() throws SQLException { - + if (isClosed) + return; + //TODO: release all resources + isClosed = true; } @Override public boolean isClosed() throws SQLException { - return false; + return isClosed; } @Override public DatabaseMetaData getMetaData() throws SQLException { - //TODO: RestfulDatabaseMetaData is not implemented - return new RestfulDatabaseMetaData(); - } - - @Override - public void setReadOnly(boolean readOnly) throws SQLException { - - } - - @Override - public boolean isReadOnly() throws SQLException { - return false; - } - - @Override - public void setCatalog(String catalog) throws SQLException { - - } - - @Override - public String getCatalog() throws SQLException { - return null; - } - - @Override - public void setTransactionIsolation(int level) throws SQLException { - //transaction is not supported - throw new SQLFeatureNotSupportedException("transactions are not supported"); - } - - /** - * - */ - @Override - public int getTransactionIsolation() throws SQLException { - //Connection.TRANSACTION_NONE specifies that transactions are not supported. - return Connection.TRANSACTION_NONE; - } - - @Override - public SQLWarning getWarnings() throws SQLException { - //TODO: getWarnings not implemented - return null; - } - - @Override - public void clearWarnings() throws SQLException { - throw new SQLFeatureNotSupportedException("clearWarnings not supported."); - } - - @Override - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - return null; - } - - @Override - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - return null; - } - - @Override - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - return null; - } - - @Override - public Map> getTypeMap() throws SQLException { - return null; - } - - @Override - public void setTypeMap(Map> map) throws SQLException { - - } - - @Override - public void setHoldability(int holdability) throws SQLException { - - } - - @Override - public int getHoldability() throws SQLException { - return 0; - } - - @Override - public Savepoint setSavepoint() throws SQLException { - return null; - } - - @Override - public Savepoint setSavepoint(String name) throws SQLException { - return null; - } - - @Override - public void rollback(Savepoint savepoint) throws SQLException { - - } - - @Override - public void releaseSavepoint(Savepoint savepoint) throws SQLException { - - } - - @Override - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - return null; - } - - @Override - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - return null; - } - - @Override - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - return null; - } - - @Override - public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - return null; - } - - @Override - public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - return null; - } - - @Override - public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - return null; - } - - @Override - public Clob createClob() throws SQLException { - //TODO: not supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Blob createBlob() throws SQLException { - //TODO: not supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public NClob createNClob() throws SQLException { - //TODO: not supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public SQLXML createSQLXML() throws SQLException { - //TODO: not supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public boolean isValid(int timeout) throws SQLException { - return false; - } - - @Override - public void setClientInfo(String name, String value) throws SQLClientInfoException { - - } - - @Override - public void setClientInfo(Properties properties) throws SQLClientInfoException { - - } - - @Override - public String getClientInfo(String name) throws SQLException { - return null; - } - - @Override - public Properties getClientInfo() throws SQLException { - return null; - } - - @Override - public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - //TODO: not supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - //TODO: not supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void setSchema(String schema) throws SQLException { - - } - - @Override - public String getSchema() throws SQLException { - return null; - } - - @Override - public void abort(Executor executor) throws SQLException { - - } - - @Override - public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - - } - - @Override - public int getNetworkTimeout() throws SQLException { - return 0; - } - - @Override - public T unwrap(Class iface) throws SQLException { - try { - return iface.cast(this); - } catch (ClassCastException cce) { - throw new SQLException("Unable to unwrap to " + iface.toString()); - } - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + ; - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return iface.isInstance(this); + return this.metadata; } public String getHost() { @@ -320,10 +69,6 @@ public class RestfulConnection implements Connection { return port; } - public Properties getProps() { - return props; - } - public String getDatabase() { return database; } @@ -331,4 +76,4 @@ public class RestfulConnection implements Connection { public String getUrl() { return url; } -} +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java index 2b4d7899fa6dc6fbd45e01aacd14e99b3a6fb536..d108f46a796bbae350f942a17f90efa29cc3021e 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java @@ -1,886 +1,87 @@ package com.taosdata.jdbc.rs; -import java.sql.*; +import com.taosdata.jdbc.AbstractDatabaseMetaData; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; -public class RestfulDatabaseMetaData implements DatabaseMetaData { +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; - @Override - public boolean allProceduresAreCallable() throws SQLException { - return false; - } +public class RestfulDatabaseMetaData extends AbstractDatabaseMetaData { - @Override - public boolean allTablesAreSelectable() throws SQLException { - return false; + private final String url; + private final String userName; + private final Connection connection; + + public RestfulDatabaseMetaData(String url, String userName, Connection connection) { + this.url = url; + this.userName = userName; + this.connection = connection; } @Override public String getURL() throws SQLException { - return null; + return this.url; } @Override public String getUserName() throws SQLException { - return null; - } - - @Override - public boolean isReadOnly() throws SQLException { - return false; - } - - @Override - public boolean nullsAreSortedHigh() throws SQLException { - return false; - } - - @Override - public boolean nullsAreSortedLow() throws SQLException { - return false; - } - - @Override - public boolean nullsAreSortedAtStart() throws SQLException { - return false; - } - - @Override - public boolean nullsAreSortedAtEnd() throws SQLException { - return false; - } - - @Override - public String getDatabaseProductName() throws SQLException { - return null; - } - - @Override - public String getDatabaseProductVersion() throws SQLException { - return null; + return this.userName; } @Override public String getDriverName() throws SQLException { - return null; - } - - @Override - public String getDriverVersion() throws SQLException { - return null; - } - - @Override - public int getDriverMajorVersion() { - return 0; - } - - @Override - public int getDriverMinorVersion() { - return 0; - } - - @Override - public boolean usesLocalFiles() throws SQLException { - return false; - } - - @Override - public boolean usesLocalFilePerTable() throws SQLException { - return false; - } - - @Override - public boolean supportsMixedCaseIdentifiers() throws SQLException { - return false; - } - - @Override - public boolean storesUpperCaseIdentifiers() throws SQLException { - return false; - } - - @Override - public boolean storesLowerCaseIdentifiers() throws SQLException { - return false; - } - - @Override - public boolean storesMixedCaseIdentifiers() throws SQLException { - return false; - } - - @Override - public boolean supportsMixedCaseQuotedIdentifiers() throws SQLException { - return false; - } - - @Override - public boolean storesUpperCaseQuotedIdentifiers() throws SQLException { - return false; - } - - @Override - public boolean storesLowerCaseQuotedIdentifiers() throws SQLException { - return false; - } - - @Override - public boolean storesMixedCaseQuotedIdentifiers() throws SQLException { - return false; - } - - @Override - public String getIdentifierQuoteString() throws SQLException { - return null; - } - - @Override - public String getSQLKeywords() throws SQLException { - return null; - } - - @Override - public String getNumericFunctions() throws SQLException { - return null; - } - - @Override - public String getStringFunctions() throws SQLException { - return null; - } - - @Override - public String getSystemFunctions() throws SQLException { - return null; - } - - @Override - public String getTimeDateFunctions() throws SQLException { - return null; - } - - @Override - public String getSearchStringEscape() throws SQLException { - return null; - } - - @Override - public String getExtraNameCharacters() throws SQLException { - return null; - } - - @Override - public boolean supportsAlterTableWithAddColumn() throws SQLException { - return false; - } - - @Override - public boolean supportsAlterTableWithDropColumn() throws SQLException { - return false; - } - - @Override - public boolean supportsColumnAliasing() throws SQLException { - return false; - } - - @Override - public boolean nullPlusNonNullIsNull() throws SQLException { - return false; - } - - @Override - public boolean supportsConvert() throws SQLException { - return false; - } - - @Override - public boolean supportsConvert(int fromType, int toType) throws SQLException { - return false; - } - - @Override - public boolean supportsTableCorrelationNames() throws SQLException { - return false; - } - - @Override - public boolean supportsDifferentTableCorrelationNames() throws SQLException { - return false; - } - - @Override - public boolean supportsExpressionsInOrderBy() throws SQLException { - return false; - } - - @Override - public boolean supportsOrderByUnrelated() throws SQLException { - return false; - } - - @Override - public boolean supportsGroupBy() throws SQLException { - return false; - } - - @Override - public boolean supportsGroupByUnrelated() throws SQLException { - return false; - } - - @Override - public boolean supportsGroupByBeyondSelect() throws SQLException { - return false; - } - - @Override - public boolean supportsLikeEscapeClause() throws SQLException { - return false; - } - - @Override - public boolean supportsMultipleResultSets() throws SQLException { - return false; - } - - @Override - public boolean supportsMultipleTransactions() throws SQLException { - return false; - } - - @Override - public boolean supportsNonNullableColumns() throws SQLException { - return false; - } - - @Override - public boolean supportsMinimumSQLGrammar() throws SQLException { - return false; - } - - @Override - public boolean supportsCoreSQLGrammar() throws SQLException { - return false; - } - - @Override - public boolean supportsExtendedSQLGrammar() throws SQLException { - return false; - } - - @Override - public boolean supportsANSI92EntryLevelSQL() throws SQLException { - return false; - } - - @Override - public boolean supportsANSI92IntermediateSQL() throws SQLException { - return false; - } - - @Override - public boolean supportsANSI92FullSQL() throws SQLException { - return false; - } - - @Override - public boolean supportsIntegrityEnhancementFacility() throws SQLException { - return false; - } - - @Override - public boolean supportsOuterJoins() throws SQLException { - return false; - } - - @Override - public boolean supportsFullOuterJoins() throws SQLException { - return false; - } - - @Override - public boolean supportsLimitedOuterJoins() throws SQLException { - return false; - } - - @Override - public String getSchemaTerm() throws SQLException { - return null; - } - - @Override - public String getProcedureTerm() throws SQLException { - return null; - } - - @Override - public String getCatalogTerm() throws SQLException { - return null; - } - - @Override - public boolean isCatalogAtStart() throws SQLException { - return false; - } - - @Override - public String getCatalogSeparator() throws SQLException { - return null; - } - - @Override - public boolean supportsSchemasInDataManipulation() throws SQLException { - return false; - } - - @Override - public boolean supportsSchemasInProcedureCalls() throws SQLException { - return false; - } - - @Override - public boolean supportsSchemasInTableDefinitions() throws SQLException { - return false; - } - - @Override - public boolean supportsSchemasInIndexDefinitions() throws SQLException { - return false; - } - - @Override - public boolean supportsSchemasInPrivilegeDefinitions() throws SQLException { - return false; - } - - @Override - public boolean supportsCatalogsInDataManipulation() throws SQLException { - return false; - } - - @Override - public boolean supportsCatalogsInProcedureCalls() throws SQLException { - return false; - } - - @Override - public boolean supportsCatalogsInTableDefinitions() throws SQLException { - return false; - } - - @Override - public boolean supportsCatalogsInIndexDefinitions() throws SQLException { - return false; - } - - @Override - public boolean supportsCatalogsInPrivilegeDefinitions() throws SQLException { - return false; - } - - @Override - public boolean supportsPositionedDelete() throws SQLException { - return false; - } - - @Override - public boolean supportsPositionedUpdate() throws SQLException { - return false; - } - - @Override - public boolean supportsSelectForUpdate() throws SQLException { - return false; - } - - @Override - public boolean supportsStoredProcedures() throws SQLException { - return false; - } - - @Override - public boolean supportsSubqueriesInComparisons() throws SQLException { - return false; - } - - @Override - public boolean supportsSubqueriesInExists() throws SQLException { - return false; - } - - @Override - public boolean supportsSubqueriesInIns() throws SQLException { - return false; - } - - @Override - public boolean supportsSubqueriesInQuantifieds() throws SQLException { - return false; - } - - @Override - public boolean supportsCorrelatedSubqueries() throws SQLException { - return false; - } - - @Override - public boolean supportsUnion() throws SQLException { - return false; - } - - @Override - public boolean supportsUnionAll() throws SQLException { - return false; - } - - @Override - public boolean supportsOpenCursorsAcrossCommit() throws SQLException { - return false; - } - - @Override - public boolean supportsOpenCursorsAcrossRollback() throws SQLException { - return false; - } - - @Override - public boolean supportsOpenStatementsAcrossCommit() throws SQLException { - return false; - } - - @Override - public boolean supportsOpenStatementsAcrossRollback() throws SQLException { - return false; - } - - @Override - public int getMaxBinaryLiteralLength() throws SQLException { - return 0; - } - - @Override - public int getMaxCharLiteralLength() throws SQLException { - return 0; - } - - @Override - public int getMaxColumnNameLength() throws SQLException { - return 0; - } - - @Override - public int getMaxColumnsInGroupBy() throws SQLException { - return 0; - } - - @Override - public int getMaxColumnsInIndex() throws SQLException { - return 0; - } - - @Override - public int getMaxColumnsInOrderBy() throws SQLException { - return 0; - } - - @Override - public int getMaxColumnsInSelect() throws SQLException { - return 0; - } - - @Override - public int getMaxColumnsInTable() throws SQLException { - return 0; - } - - @Override - public int getMaxConnections() throws SQLException { - return 0; - } - - @Override - public int getMaxCursorNameLength() throws SQLException { - return 0; - } - - @Override - public int getMaxIndexLength() throws SQLException { - return 0; - } - - @Override - public int getMaxSchemaNameLength() throws SQLException { - return 0; - } - - @Override - public int getMaxProcedureNameLength() throws SQLException { - return 0; - } - - @Override - public int getMaxCatalogNameLength() throws SQLException { - return 0; - } - - @Override - public int getMaxRowSize() throws SQLException { - return 0; - } - - @Override - public boolean doesMaxRowSizeIncludeBlobs() throws SQLException { - return false; - } - - @Override - public int getMaxStatementLength() throws SQLException { - return 0; - } - - @Override - public int getMaxStatements() throws SQLException { - return 0; - } - - @Override - public int getMaxTableNameLength() throws SQLException { - return 0; - } - - @Override - public int getMaxTablesInSelect() throws SQLException { - return 0; - } - - @Override - public int getMaxUserNameLength() throws SQLException { - return 0; - } - - @Override - public int getDefaultTransactionIsolation() throws SQLException { - return 0; - } - - @Override - public boolean supportsTransactions() throws SQLException { - return false; - } - - @Override - public boolean supportsTransactionIsolationLevel(int level) throws SQLException { - return false; - } - - @Override - public boolean supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { - return false; - } - - @Override - public boolean supportsDataManipulationTransactionsOnly() throws SQLException { - return false; - } - - @Override - public boolean dataDefinitionCausesTransactionCommit() throws SQLException { - return false; - } - - @Override - public boolean dataDefinitionIgnoredInTransactions() throws SQLException { - return false; - } - - @Override - public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) throws SQLException { - return null; - } - - @Override - public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, String columnNamePattern) throws SQLException { - return null; + return RestfulDriver.class.getName(); } @Override public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { - return null; - } - - @Override - public ResultSet getSchemas() throws SQLException { - return null; + if (connection == null || connection.isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + return super.getTables(catalog, schemaPattern, tableNamePattern, types, connection); } @Override public ResultSet getCatalogs() throws SQLException { - return null; + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getCatalogs(connection); } @Override public ResultSet getTableTypes() throws SQLException { - return null; + if (connection == null || connection.isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + return super.getTableTypes(); } @Override public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { - return null; - } - - @Override - public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) throws SQLException { - return null; - } - - @Override - public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { - return null; - } - - @Override - public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) throws SQLException { - return null; - } - - @Override - public ResultSet getVersionColumns(String catalog, String schema, String table) throws SQLException { - return null; + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern, connection); } @Override public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { - return null; - } - - @Override - public ResultSet getImportedKeys(String catalog, String schema, String table) throws SQLException { - return null; - } - - @Override - public ResultSet getExportedKeys(String catalog, String schema, String table) throws SQLException { - return null; - } - - @Override - public ResultSet getCrossReference(String parentCatalog, String parentSchema, String parentTable, String foreignCatalog, String foreignSchema, String foreignTable) throws SQLException { - return null; - } - - @Override - public ResultSet getTypeInfo() throws SQLException { - return null; - } - - @Override - public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) throws SQLException { - return null; - } - - @Override - public boolean supportsResultSetType(int type) throws SQLException { - return false; - } - - @Override - public boolean supportsResultSetConcurrency(int type, int concurrency) throws SQLException { - return false; - } - - @Override - public boolean ownUpdatesAreVisible(int type) throws SQLException { - return false; - } - - @Override - public boolean ownDeletesAreVisible(int type) throws SQLException { - return false; - } - - @Override - public boolean ownInsertsAreVisible(int type) throws SQLException { - return false; - } - - @Override - public boolean othersUpdatesAreVisible(int type) throws SQLException { - return false; - } - - @Override - public boolean othersDeletesAreVisible(int type) throws SQLException { - return false; - } - - @Override - public boolean othersInsertsAreVisible(int type) throws SQLException { - return false; - } - - @Override - public boolean updatesAreDetected(int type) throws SQLException { - return false; - } - - @Override - public boolean deletesAreDetected(int type) throws SQLException { - return false; - } - - @Override - public boolean insertsAreDetected(int type) throws SQLException { - return false; - } - - @Override - public boolean supportsBatchUpdates() throws SQLException { - return false; - } - - @Override - public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) throws SQLException { - return null; + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getPrimaryKeys(catalog, schema, table, connection); } @Override public Connection getConnection() throws SQLException { - return null; - } - - @Override - public boolean supportsSavepoints() throws SQLException { - return false; - } - - @Override - public boolean supportsNamedParameters() throws SQLException { - return false; - } - - @Override - public boolean supportsMultipleOpenResults() throws SQLException { - return false; - } - - @Override - public boolean supportsGetGeneratedKeys() throws SQLException { - return false; - } - - @Override - public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) throws SQLException { - return null; + return this.connection; } @Override public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { - return null; + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getSuperTables(catalog, schemaPattern, tableNamePattern, connection); } - @Override - public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, String attributeNamePattern) throws SQLException { - return null; - } - - @Override - public boolean supportsResultSetHoldability(int holdability) throws SQLException { - return false; - } - - @Override - public int getResultSetHoldability() throws SQLException { - return 0; - } - - @Override - public int getDatabaseMajorVersion() throws SQLException { - return 0; - } - - @Override - public int getDatabaseMinorVersion() throws SQLException { - return 0; - } - - @Override - public int getJDBCMajorVersion() throws SQLException { - return 0; - } - - @Override - public int getJDBCMinorVersion() throws SQLException { - return 0; - } - - @Override - public int getSQLStateType() throws SQLException { - return 0; - } - - @Override - public boolean locatorsUpdateCopy() throws SQLException { - return false; - } - - @Override - public boolean supportsStatementPooling() throws SQLException { - return false; - } - - @Override - public RowIdLifetime getRowIdLifetime() throws SQLException { - return null; - } - - @Override - public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { - return null; - } - - @Override - public boolean supportsStoredFunctionsUsingCallSyntax() throws SQLException { - return false; - } - - @Override - public boolean autoCommitFailureClosesAllResultSets() throws SQLException { - return false; - } - - @Override - public ResultSet getClientInfoProperties() throws SQLException { - return null; - } - - @Override - public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) throws SQLException { - return null; - } - - @Override - public ResultSet getFunctionColumns(String catalog, String schemaPattern, String functionNamePattern, String columnNamePattern) throws SQLException { - return null; - } - - @Override - public ResultSet getPseudoColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { - return null; - } - - @Override - public boolean generatedKeyAlwaysReturned() throws SQLException { - return false; - } - - @Override - public T unwrap(Class iface) throws SQLException { - return null; - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return false; - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index 9e87cfa68015ece9fdb0208b290daaecf75b2191..d45ca0c3a0bc4ea596b9e9577db33e8aa1bee696 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -2,16 +2,18 @@ package com.taosdata.jdbc.rs; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; -import com.taosdata.jdbc.AbstractTaosDriver; +import com.taosdata.jdbc.AbstractDriver; import com.taosdata.jdbc.TSDBConstants; import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.rs.util.HttpClientPoolUtil; +import com.taosdata.jdbc.utils.HttpClientPoolUtil; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.sql.*; import java.util.Properties; import java.util.logging.Logger; -public class RestfulDriver extends AbstractTaosDriver { +public class RestfulDriver extends AbstractDriver { private static final String URL_PREFIX = "jdbc:TAOS-RS://"; @@ -33,7 +35,7 @@ public class RestfulDriver extends AbstractTaosDriver { return null; Properties props = parseURL(url, info); - String host = props.getProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); + String host = props.getProperty(TSDBDriver.PROPERTY_KEY_HOST); String port = props.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "6041"); String database = props.containsKey(TSDBDriver.PROPERTY_KEY_DBNAME) ? props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME) : null; @@ -41,9 +43,20 @@ public class RestfulDriver extends AbstractTaosDriver { + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + props.getProperty(TSDBDriver.PROPERTY_KEY_USER) + "/" + props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD) + ""; + try { + String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), "UTF-8"); + String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), "UTF-8"); + loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + user + "/" + password + ""; + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + String result = HttpClientPoolUtil.execute(loginUrl); JSONObject jsonResult = JSON.parseObject(result); String status = jsonResult.getString("status"); + String token = jsonResult.getString("desc"); + HttpClientPoolUtil.token = token; if (!status.equals("succ")) { throw new SQLException(jsonResult.getString("desc")); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java new file mode 100644 index 0000000000000000000000000000000000000000..3a0ff56dd71979f21b24f68b41ea9d7bb994586e --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java @@ -0,0 +1,465 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.*; +import java.util.Calendar; + +public class RestfulPreparedStatement extends RestfulStatement implements PreparedStatement { + + private ParameterMetaData parameterMetaData; + private final String rawSql; + private Object[] parameters; + private boolean isPrepared; + + public RestfulPreparedStatement(RestfulConnection conn, String database, String sql) { + super(conn, database); + this.rawSql = sql; + if (sql.contains("?")) { + int parameterCnt = 0; + for (int i = 0; i < sql.length(); i++) { + if ('?' == sql.charAt(i)) { + parameterCnt++; + } + } + parameters = new Object[parameterCnt]; + this.isPrepared = true; + } + //TODO: build parameterMetaData + } + + @Override + public ResultSet executeQuery() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isPrepared) + return executeQuery(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + return executeQuery(sql); + } + + @Override + public int executeUpdate() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isPrepared) + return executeUpdate(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + return executeUpdate(sql); + } + + private String getNativeSql(String rawSql) throws SQLException { + String sql = rawSql; + for (int i = 0; i < parameters.length; ++i) { + Object para = parameters[i]; + if (para != null) { + String paraStr = para.toString(); + if (para instanceof Timestamp || para instanceof String) { + paraStr = "'" + paraStr + "'"; + } + sql = sql.replaceFirst("[?]", paraStr); + } else { + sql = sql.replaceFirst("[?]", "NULL"); + } + } + clearParameters(); + return sql; + } + + @Override + public void setNull(int parameterIndex, int sqlType) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, "NULL"); + } + + @Override + public void setBoolean(int parameterIndex, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setByte(int parameterIndex, byte x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setShort(int parameterIndex, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setInt(int parameterIndex, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, x); + } + + @Override + public void setLong(int parameterIndex, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setFloat(int parameterIndex, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setDouble(int parameterIndex, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setString(int parameterIndex, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setBytes(int parameterIndex, byte[] x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setDate(int parameterIndex, Date x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTime(int parameterIndex, Time x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void clearParameters() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + this.parameters = new Object[parameters.length]; + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setObject(int parameterIndex, Object x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + if (parameterIndex < 1 && parameterIndex >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + parameters[parameterIndex - 1] = x; + + } + + @Override + public boolean execute() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isPrepared) + return execute(this.rawSql); + final String sql = getNativeSql(rawSql); + return execute(sql); + } + + @Override + public void addBatch() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + final String sql = getNativeSql(this.rawSql); + addBatch(sql); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setRef(int parameterIndex, Ref x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBlob(int parameterIndex, Blob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setClob(int parameterIndex, Clob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setArray(int parameterIndex, Array x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setURL(int parameterIndex, URL x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public ParameterMetaData getParameterMetaData() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); +// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + return this.parameterMetaData; + } + + @Override + public void setRowId(int parameterIndex, RowId x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNString(int parameterIndex, String value) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNClob(int parameterIndex, NClob value) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setClob(int parameterIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNClob(int parameterIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index c536ae4a8957519cdcb8d64f95fbdf2934624fff..9d394b8b038917d637ab43826d57e4d79b1746c7 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -1,48 +1,166 @@ package com.taosdata.jdbc.rs; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.taosdata.jdbc.AbstractResultSet; import com.taosdata.jdbc.TSDBConstants; -import org.apache.commons.lang3.StringUtils; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; -import java.io.InputStream; -import java.io.Reader; -import java.math.BigDecimal; -import java.net.URL; import java.sql.*; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Map; +import java.util.List; -public class RestfulResultSet implements ResultSet { - private boolean isClosed = false; +public class RestfulResultSet extends AbstractResultSet implements ResultSet { + private volatile boolean isClosed; private int pos = -1; - private ArrayList> data; - private ArrayList fields; - public RestfulResultSet(String str, String fieldData) { - data = new ArrayList<>(); - str = str.substring(2, str.length() - 2); - ArrayList strTemp = new ArrayList<>(Arrays.asList(str.split("],\\["))); - for (String s : strTemp) { - ArrayList curr = new ArrayList<>(Arrays.asList(s.split(","))); - data.add(curr); + private final String database; + private final Statement statement; + // data + private ArrayList> resultSet; + // meta + private ArrayList columnNames; + private ArrayList columns; + private RestfulResultSetMetaData metaData; + + /** + * 由一个result的Json构造结果集，对应执行show databases, show tables等这些语句，返回结果集，但无法获取结果集对应的meta，统一当成String处理 + * + * @param resultJson: 包含data信息的结果集，有sql返回的结果集 + ***/ + public RestfulResultSet(String database, Statement statement, JSONObject resultJson) throws SQLException { + this.database = database; + this.statement = statement; + // column metadata + JSONArray columnMeta = resultJson.getJSONArray("column_meta"); + columnNames = new ArrayList<>(); + columns = new ArrayList<>(); + for (int colIndex = 0; colIndex < columnMeta.size(); colIndex++) { + JSONArray col = columnMeta.getJSONArray(colIndex); + String col_name = col.getString(0); + int col_type = TSDBConstants.taosType2JdbcType(col.getInteger(1)); + int col_length = col.getInteger(2); + columnNames.add(col_name); + columns.add(new Field(col_name, col_type, col_length, "")); } - if (!StringUtils.isBlank(fieldData)) { - fields = new ArrayList<>(); - fieldData = fieldData.substring(2, fieldData.length() - 2); - ArrayList fieldTemp = new ArrayList<>(Arrays.asList(fieldData.split("],\\["))); - for (String s : fieldTemp) { - String curr = Arrays.asList(s.split(",")).get(0); - fields.add(curr.substring(1, curr.length() - 1)); // 去掉双引号 + this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + + // row data + JSONArray data = resultJson.getJSONArray("data"); + resultSet = new ArrayList<>(); + for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { + ArrayList row = new ArrayList(); + JSONArray jsonRow = data.getJSONArray(rowIndex); + for (int colIndex = 0; colIndex < jsonRow.size(); colIndex++) { + row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).type)); } + resultSet.add(row); + } + + /* + int columnIndex = 0; + for (; columnIndex < data.size(); columnIndex++) { + ArrayList oneRow = new ArrayList<>(); + JSONArray one = data.getJSONArray(columnIndex); + for (int j = 0; j < one.size(); j++) { + oneRow.add(one.getString(j)); + } + resultSet.add(oneRow); + } + + // column only names + JSONArray head = resultJson.getJSONArray("head"); + for (int i = 0; i < head.size(); i++) { + String name = head.getString(i); + columnNames.add(name); + columns.add(new Field(name, "", 0, "")); + } + this.metaData = new RestfulResultSetMetaData(this.database, columns, this); + */ + } + + private Object parseColumnData(JSONArray row, int colIndex, int sqlType) { + switch (sqlType) { + case Types.NULL: + return null; + case Types.BOOLEAN: + return row.getBoolean(colIndex); + case Types.TINYINT: + case Types.SMALLINT: + return row.getShort(colIndex); + case Types.INTEGER: + return row.getInteger(colIndex); + case Types.BIGINT: + return row.getBigInteger(colIndex); + case Types.FLOAT: + return row.getFloat(colIndex); + case Types.DOUBLE: + return row.getDouble(colIndex); + case Types.TIMESTAMP: + return new Timestamp(row.getDate(colIndex).getTime()); + case Types.BINARY: + case Types.NCHAR: + default: + return row.getString(colIndex); + } + } + +// /** +// * 由多个resultSet的JSON构造结果集 +// * +// * @param resultJson: 包含data信息的结果集，有sql返回的结果集 +// * @param fieldJson: 包含多个（最多2个）meta信息的结果集，有describe xxx +// **/ +// public RestfulResultSet(String database, Statement statement, JSONObject resultJson, List fieldJson) throws SQLException { +// this(database, statement, resultJson); +// ArrayList newColumns = new ArrayList<>(); +// +// for (Field column : columns) { +// Field field = findField(column.name, fieldJson); +// if (field != null) { +// newColumns.add(field); +// } else { +// newColumns.add(column); +// } +// } +// this.columns = newColumns; +// this.metaData = new RestfulResultSetMetaData(this.database, this.columns, this); +// } + +// public Field findField(String columnName, List fieldJsonList) { +// for (JSONObject fieldJSON : fieldJsonList) { +// JSONArray fieldDataJson = fieldJSON.getJSONArray("data"); +// for (int i = 0; i < fieldDataJson.size(); i++) { +// JSONArray field = fieldDataJson.getJSONArray(i); +// if (columnName.equalsIgnoreCase(field.getString(0))) { +// return new Field(field.getString(0), field.getString(1), field.getInteger(2), field.getString(3)); +// } +// } +// } +// return null; +// } + + public class Field { + String name; + int type; + int length; + String note; + + public Field(String name, int type, int length, String note) { + this.name = name; + this.type = type; + this.length = length; + this.note = note; } } @Override public boolean next() throws SQLException { - if (isClosed) throw new SQLException(TSDBConstants.WrapErrMsg("Result is Closed!!!")); - if (pos < data.size() - 1) { - pos++; + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + pos++; + if (pos <= resultSet.size() - 1) { return true; } return false; @@ -50,1131 +168,295 @@ public class RestfulResultSet implements ResultSet { @Override public void close() throws SQLException { - this.isClosed = true; + synchronized (RestfulResultSet.class) { + this.isClosed = true; + } } @Override public boolean wasNull() throws SQLException { - return data.isEmpty(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return resultSet.isEmpty(); } @Override public String getString(int columnIndex) throws SQLException { - if (columnIndex > data.get(pos).size()) { - throw new SQLException(TSDBConstants.WrapErrMsg("Column Index out of range, " + columnIndex + " > " + data.get(pos).size())); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + if (columnIndex > resultSet.get(pos).size()) { + throw new SQLException(TSDBConstants.WrapErrMsg("Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size())); } - return data.get(pos).get(columnIndex - 1); + + columnIndex = getTrueColumnIndex(columnIndex); + return resultSet.get(pos).get(columnIndex).toString(); } @Override public boolean getBoolean(int columnIndex) throws SQLException { - String result = getString(columnIndex); - if (!(result.equals("true") || result.equals("false"))) { - throw new SQLException("not boolean value"); - } - return result.equals("true"); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public byte getByte(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + columnIndex = getTrueColumnIndex(columnIndex); + int result = getInt(columnIndex); + return result == 0 ? false : true; } @Override public short getShort(int columnIndex) throws SQLException { - return Short.parseShort(getString(columnIndex)); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + columnIndex = getTrueColumnIndex(columnIndex); + return Short.parseShort(resultSet.get(pos).get(columnIndex).toString()); } @Override public int getInt(int columnIndex) throws SQLException { - String result = getString(columnIndex); - return Integer.parseInt(result); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + columnIndex = getTrueColumnIndex(columnIndex); + return Integer.parseInt(resultSet.get(pos).get(columnIndex).toString()); } @Override public long getLong(int columnIndex) throws SQLException { - String result = getString(columnIndex); - return Long.parseLong(result); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + columnIndex = getTrueColumnIndex(columnIndex); + return Long.parseLong(resultSet.get(pos).get(columnIndex).toString()); } @Override public float getFloat(int columnIndex) throws SQLException { - String result = getString(columnIndex); - return Float.parseFloat(result); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + columnIndex = getTrueColumnIndex(columnIndex); + return Float.parseFloat(resultSet.get(pos).get(columnIndex).toString()); } @Override public double getDouble(int columnIndex) throws SQLException { - String result = getString(columnIndex); - return Double.parseDouble(result); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + columnIndex = getTrueColumnIndex(columnIndex); + return Double.parseDouble(resultSet.get(pos).get(columnIndex).toString()); } - @Override - public byte[] getBytes(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } + private int getTrueColumnIndex(int columnIndex) throws SQLException { + if (columnIndex < 1) { + throw new SQLException("Column Index out of range, " + columnIndex + " < 1"); + } - @Override - public Date getDate(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } + int numOfCols = resultSet.get(pos).size(); + if (columnIndex > numOfCols) { + throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols); + } - @Override - public Time getTime(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + return columnIndex - 1; } @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { - String strDate = getString(columnIndex); - strDate = strDate.substring(1, strDate.length() - 1); - return Timestamp.valueOf(strDate); - } - - @Override - public InputStream getAsciiStream(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public InputStream getUnicodeStream(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public InputStream getBinaryStream(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public String getString(String columnLabel) throws SQLException { - return getString(findColumn(columnLabel) + 1); - } - - @Override - public boolean getBoolean(String columnLabel) throws SQLException { - return Boolean.parseBoolean(getString(columnLabel)); - } - - @Override - public byte getByte(String columnLabel) throws SQLException { - return 0; - } - - @Override - public short getShort(String columnLabel) throws SQLException { - return Short.parseShort(getString(columnLabel)); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public int getInt(String columnLabel) throws SQLException { - return Integer.parseInt(getString(columnLabel)); - } - - @Override - public long getLong(String columnLabel) throws SQLException { - return Long.parseLong(getString(columnLabel)); - } - - @Override - public float getFloat(String columnLabel) throws SQLException { - String result = getString(columnLabel); - return Float.parseFloat(result); - } - - @Override - public double getDouble(String columnLabel) throws SQLException { - return Double.parseDouble(getString(columnLabel)); - } - - @Override - public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { - return null; - } - - @Override - public byte[] getBytes(String columnLabel) throws SQLException { - return new byte[0]; - } - - @Override - public Date getDate(String columnLabel) throws SQLException { - return null; - } - - @Override - public Time getTime(String columnLabel) throws SQLException { - return null; - } - - @Override - public Timestamp getTimestamp(String columnLabel) throws SQLException { - return Timestamp.valueOf(getString(columnLabel)); - } - - @Override - public InputStream getAsciiStream(String columnLabel) throws SQLException { - return null; - } - - @Override - public InputStream getUnicodeStream(String columnLabel) throws SQLException { - return null; - } - - @Override - public InputStream getBinaryStream(String columnLabel) throws SQLException { - return null; - } - - @Override - public SQLWarning getWarnings() throws SQLException { - return null; - //TODO: SQLFeature Not Supported -// throw new SQLFeatureNotSupportedException(); - } - - @Override - public void clearWarnings() throws SQLException { - return; - //TODO: SQLFeature Not Supported -// throw new SQLFeatureNotSupportedException(); - } - - @Override - public String getCursorName() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + columnIndex = getTrueColumnIndex(columnIndex); + String strDate = resultSet.get(pos).get(columnIndex).toString(); +// strDate = strDate.substring(1, strDate.length() - 1); + return Timestamp.valueOf(strDate); } + /*************************************************************************************************************/ @Override public ResultSetMetaData getMetaData() throws SQLException { - return new RestfulResultSetMetaData(fields); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public Object getObject(int columnIndex) throws SQLException { -// return null; - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + return this.metaData; } @Override public Object getObject(String columnLabel) throws SQLException { -// return null; - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + return getObject(findColumn(columnLabel)); } @Override public int findColumn(String columnLabel) throws SQLException { - return fields.indexOf(columnLabel); - } - - @Override - public Reader getCharacterStream(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Reader getCharacterStream(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public BigDecimal getBigDecimal(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + int columnIndex = columnNames.indexOf(columnLabel); + if (columnIndex == -1) + throw new SQLException("cannot find Column in resultSet"); + return columnIndex + 1; } @Override public boolean isBeforeFirst() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return this.pos == -1 && this.resultSet.size() != 0; } @Override public boolean isAfterLast() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + return this.pos >= resultSet.size() && this.resultSet.size() != 0; } @Override public boolean isFirst() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return this.pos == 0; } @Override public boolean isLast() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + if (this.resultSet.size() == 0) + return false; + return this.pos == (this.resultSet.size() - 1); } @Override public void beforeFirst() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + synchronized (this) { + if (this.resultSet.size() > 0) { + this.pos = -1; + } + } } @Override public void afterLast() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + synchronized (this) { + if (this.resultSet.size() > 0) { + this.pos = this.resultSet.size(); + } + } } @Override public boolean first() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + if (this.resultSet.size() == 0) + return false; + + synchronized (this) { + this.pos = 0; + } + return true; } @Override public boolean last() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + if (this.resultSet.size() == 0) + return false; + synchronized (this) { + this.pos = this.resultSet.size() - 1; + } + return true; } @Override public int getRow() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + int row; + synchronized (this) { + if (this.pos < 0 || this.pos >= this.resultSet.size()) + return 0; + row = this.pos + 1; + } + return row; } @Override public boolean absolute(int row) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + +// if (this.resultSet.size() == 0) +// return false; +// +// if (row == 0) { +// beforeFirst(); +// return false; +// } else if (row == 1) { +// return first(); +// } else if (row == -1) { +// return last(); +// } else if (row > this.resultSet.size()) { +// afterLast(); +// return false; +// } else { +// if (row < 0) { +// // adjust to reflect after end of result set +// int newRowPosition = this.resultSet.size() + row + 1; +// if (newRowPosition <= 0) { +// beforeFirst(); +// return false; +// } else { +// return absolute(newRowPosition); +// } +// } else { +// row--; // adjust for index difference +// this.pos = row; +// return true; +// } +// } + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean relative(int rows) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public boolean previous() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void setFetchDirection(int direction) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public int getFetchDirection() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void setFetchSize(int rows) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public int getFetchSize() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public int getType() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public int getConcurrency() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public boolean rowUpdated() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public boolean rowInserted() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public boolean rowDeleted() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNull(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBoolean(int columnIndex, boolean x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateByte(int columnIndex, byte x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateShort(int columnIndex, short x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateInt(int columnIndex, int x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateLong(int columnIndex, long x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateFloat(int columnIndex, float x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateDouble(int columnIndex, double x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateString(int columnIndex, String x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBytes(int columnIndex, byte[] x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateDate(int columnIndex, Date x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateTime(int columnIndex, Time x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateObject(int columnIndex, Object x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNull(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBoolean(String columnLabel, boolean x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateByte(String columnLabel, byte x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateShort(String columnLabel, short x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateInt(String columnLabel, int x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateLong(String columnLabel, long x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateFloat(String columnLabel, float x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateDouble(String columnLabel, double x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateString(String columnLabel, String x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBytes(String columnLabel, byte[] x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateDate(String columnLabel, Date x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateTime(String columnLabel, Time x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateObject(String columnLabel, Object x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void insertRow() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateRow() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public void deleteRow() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void refreshRow() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override - public void cancelRowUpdates() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } + public boolean previous() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public void moveToInsertRow() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override - public void moveToCurrentRow() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + public String getNString(int columnIndex) throws SQLException { + return getString(columnIndex); } @Override public Statement getStatement() throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Object getObject(int columnIndex, Map> map) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Ref getRef(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Blob getBlob(int columnIndex) throws SQLException { - return null; - } - - @Override - public Clob getClob(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Array getArray(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Object getObject(String columnLabel, Map> map) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Ref getRef(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Blob getBlob(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Clob getClob(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Array getArray(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Date getDate(int columnIndex, Calendar cal) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Date getDate(String columnLabel, Calendar cal) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public Time getTime(int columnIndex, Calendar cal) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Time getTime(String columnLabel, Calendar cal) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public URL getURL(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public URL getURL(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateRef(int columnIndex, Ref x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateRef(String columnLabel, Ref x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBlob(int columnIndex, Blob x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBlob(String columnLabel, Blob x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateClob(int columnIndex, Clob x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateClob(String columnLabel, Clob x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateArray(int columnIndex, Array x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateArray(String columnLabel, Array x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public RowId getRowId(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public RowId getRowId(String columnLabel) throws SQLException { - return null; - } - - @Override - public void updateRowId(int columnIndex, RowId x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateRowId(String columnLabel, RowId x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public int getHoldability() throws SQLException { -// return 0; - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + return this.statement; } @Override public boolean isClosed() throws SQLException { - return false; - //TODO: SQLFeature Not Supported -// throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNString(int columnIndex, String nString) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNString(String columnLabel, String nString) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNClob(int columnIndex, NClob nClob) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNClob(String columnLabel, NClob nClob) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public NClob getNClob(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public NClob getNClob(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public SQLXML getSQLXML(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public SQLXML getSQLXML(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public String getNString(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public String getNString(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Reader getNCharacterStream(int columnIndex) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Reader getNCharacterStream(String columnLabel) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); + return isClosed; } - @Override - public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - @Override - public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateClob(int columnIndex, Reader reader) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateClob(String columnLabel, Reader reader) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNClob(int columnIndex, Reader reader) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public void updateNClob(String columnLabel, Reader reader) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public T getObject(int columnIndex, Class type) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public T getObject(String columnLabel, Class type) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public T unwrap(Class iface) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - //TODO: SQLFeature Not Supported - throw new SQLFeatureNotSupportedException(); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java index 5dd61391bcf0f973726c7954e330bb5054c4e91f..29ba13bec107be4564a95d27e800fce53b114f96 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java @@ -1,15 +1,23 @@ package com.taosdata.jdbc.rs; +import com.taosdata.jdbc.TSDBConstants; + import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.List; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.ArrayList; public class RestfulResultSetMetaData implements ResultSetMetaData { - private List fields; + private final String database; + private ArrayList fields; + private final RestfulResultSet resultSet; - public RestfulResultSetMetaData(List fields) { + public RestfulResultSetMetaData(String database, ArrayList fields, RestfulResultSet resultSet) { + this.database = database; this.fields = fields; + this.resultSet = resultSet; } @Override @@ -29,7 +37,7 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public boolean isSearchable(int column) throws SQLException { - return false; + return true; } @Override @@ -39,67 +47,100 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int isNullable(int column) throws SQLException { - return 0; + if (column == 1) + return ResultSetMetaData.columnNoNulls; + return ResultSetMetaData.columnNullable; } @Override public boolean isSigned(int column) throws SQLException { - return false; + int type = this.fields.get(column - 1).type; + switch (type) { + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + case Types.BIGINT: + case Types.FLOAT: + case Types.DOUBLE: + return true; + default: + return false; + } } @Override public int getColumnDisplaySize(int column) throws SQLException { - return 0; + return this.fields.get(column - 1).length; } @Override public String getColumnLabel(int column) throws SQLException { - return fields.get(column - 1); + return fields.get(column - 1).name; } @Override public String getColumnName(int column) throws SQLException { - return null; + return fields.get(column - 1).name; } @Override public String getSchemaName(int column) throws SQLException { - return null; + return ""; } @Override public int getPrecision(int column) throws SQLException { - return 0; + int type = this.fields.get(column - 1).type; + switch (type) { + case Types.FLOAT: + return 5; + case Types.DOUBLE: + return 9; + case Types.BINARY: + case Types.NCHAR: + return this.fields.get(column - 1).length; + default: + return 0; + } } @Override public int getScale(int column) throws SQLException { - return 0; + int type = this.fields.get(column - 1).type; + switch (type) { + case Types.FLOAT: + return 5; + case Types.DOUBLE: + return 9; + default: + return 0; + } } @Override public String getTableName(int column) throws SQLException { - return null; + return ""; } @Override public String getCatalogName(int column) throws SQLException { - return null; + return this.database; } @Override public int getColumnType(int column) throws SQLException { - return 0; + return this.fields.get(column - 1).type; } @Override public String getColumnTypeName(int column) throws SQLException { - return null; + int type = fields.get(column - 1).type; + return TSDBConstants.jdbcType2TaosTypeName(type); } @Override public boolean isReadOnly(int column) throws SQLException { - return false; + return true; } @Override @@ -114,16 +155,43 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public String getColumnClassName(int column) throws SQLException { - return null; + int type = this.fields.get(column - 1).type; + String columnClassName = ""; + switch (type) { + case Types.BOOLEAN: + return Boolean.class.getName(); + case Types.TINYINT: + case Types.SMALLINT: + return Short.class.getName(); + case Types.INTEGER: + return Integer.class.getName(); + case Types.BIGINT: + return Long.class.getName(); + case Types.FLOAT: + return Float.class.getName(); + case Types.DOUBLE: + return Double.class.getName(); + case Types.TIMESTAMP: + return Timestamp.class.getName(); + case Types.BINARY: + case Types.NCHAR: + return String.class.getName(); + } + return columnClassName; } @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } + } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 30b56638d880137b7a3394e5282a3edf5178d2b0..d60940d87762340203e54fb7df23579c2f93d510 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -1,291 +1,222 @@ package com.taosdata.jdbc.rs; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.taosdata.jdbc.AbstractStatement; import com.taosdata.jdbc.TSDBConstants; -import com.taosdata.jdbc.rs.util.HttpClientPoolUtil; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; +import com.taosdata.jdbc.utils.HttpClientPoolUtil; import com.taosdata.jdbc.utils.SqlSyntaxValidator; import java.sql.*; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; -public class RestfulStatement implements Statement { +public class RestfulStatement extends AbstractStatement { private boolean closed; private String database; private final RestfulConnection conn; - public RestfulStatement(RestfulConnection c, String database) { - this.conn = c; + private volatile RestfulResultSet resultSet; + private volatile int affectedRows; + + public RestfulStatement(RestfulConnection conn, String database) { + this.conn = conn; this.database = database; } + protected String[] parseTableIdentifier(String sql) { + sql = sql.trim().toLowerCase(); + String[] ret = null; + if (sql.contains("where")) + sql = sql.substring(0, sql.indexOf("where")); + if (sql.contains("interval")) + sql = sql.substring(0, sql.indexOf("interval")); + if (sql.contains("fill")) + sql = sql.substring(0, sql.indexOf("fill")); + if (sql.contains("sliding")) + sql = sql.substring(0, sql.indexOf("sliding")); + if (sql.contains("group by")) + sql = sql.substring(0, sql.indexOf("group by")); + if (sql.contains("order by")) + sql = sql.substring(0, sql.indexOf("order by")); + if (sql.contains("slimit")) + sql = sql.substring(0, sql.indexOf("slimit")); + if (sql.contains("limit")) + sql = sql.substring(0, sql.indexOf("limit")); + // parse + if (sql.contains("from")) { + sql = sql.substring(sql.indexOf("from") + 4).trim(); + return Arrays.asList(sql.split(",")).stream() + .map(tableIdentifier -> { + tableIdentifier = tableIdentifier.trim(); + if (tableIdentifier.contains(" ")) + tableIdentifier = tableIdentifier.substring(0, tableIdentifier.indexOf(" ")); + return tableIdentifier; + }).collect(Collectors.joining(",")).split(","); + } + return ret; + } + @Override public ResultSet executeQuery(String sql) throws SQLException { if (isClosed()) - throw new SQLException("statement already closed"); - if (!SqlSyntaxValidator.isSelectSql(sql)) - throw new SQLException("not a select sql for executeQuery: " + sql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - String result = HttpClientPoolUtil.execute(url, sql); - String fields = ""; - List words = Arrays.asList(sql.split(" ")); - if (words.get(0).equalsIgnoreCase("select")) { - int index = 0; - if (words.contains("from")) { - index = words.indexOf("from"); - } - if (words.contains("FROM")) { - index = words.indexOf("FROM"); - } - fields = HttpClientPoolUtil.execute(url, "DESCRIBE " + words.get(index + 1)); - } - - JSONObject jsonObject = JSON.parseObject(result); - if (jsonObject.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + - jsonObject.getString("desc") + "\n" + - "error code: " + jsonObject.getString("code"))); - } - String dataStr = jsonObject.getString("data"); - if ("use".equalsIgnoreCase(fields.split(" ")[0])) { - return new RestfulResultSet(dataStr, ""); - } - - JSONObject jsonField = JSON.parseObject(fields); - if (jsonField == null) { - return new RestfulResultSet(dataStr, ""); - } - if (jsonField.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + - jsonField.getString("desc") + "\n" + - "error code: " + jsonField.getString("code"))); + if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { + return executeOneQuery(url, sql); } - String fieldData = jsonField.getString("data"); - return new RestfulResultSet(dataStr, fieldData); +// if (this.database == null || this.database.isEmpty()) +// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE); + HttpClientPoolUtil.execute(url, "use " + this.database); + return executeOneQuery(url, sql); } @Override public int executeUpdate(String sql) throws SQLException { if (isClosed()) - throw new SQLException("statement already closed"); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) - throw new SQLException("not a valid sql for executeUpdate: " + sql); - - if (this.database == null) - throw new SQLException("Database not specified or available"); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql); final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); - String result = HttpClientPoolUtil.execute(url, sql); - JSONObject jsonObject = JSON.parseObject(result); - if (jsonObject.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + - jsonObject.getString("desc") + "\n" + - "error code: " + jsonObject.getString("code"))); + if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { + return executeOneUpdate(url, sql); } - return Integer.parseInt(jsonObject.getString("rows")); - } - @Override - public void close() throws SQLException { - this.closed = true; - } - - @Override - public int getMaxFieldSize() throws SQLException { - return 0; - } - - @Override - public void setMaxFieldSize(int max) throws SQLException { - - } - - @Override - public int getMaxRows() throws SQLException { - return 0; - } - - @Override - public void setMaxRows(int max) throws SQLException { - - } - - @Override - public void setEscapeProcessing(boolean enable) throws SQLException { +// if (this.database == null || this.database.isEmpty()) +// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE); + HttpClientPoolUtil.execute(url, "use " + this.database); + return executeOneUpdate(url, sql); } @Override - public int getQueryTimeout() throws SQLException { - return 0; - } - - @Override - public void setQueryTimeout(int seconds) throws SQLException { - - } - - @Override - public void cancel() throws SQLException { - + public void close() throws SQLException { + synchronized (RestfulStatement.class) { + if (!isClosed()) + this.closed = true; + } } @Override - public SQLWarning getWarnings() throws SQLException { - //TODO: getWarnings not Implemented - return null; - } + public boolean execute(String sql) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!SqlSyntaxValidator.isValidForExecute(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "not a valid sql for execute: " + sql); - @Override - public void clearWarnings() throws SQLException { + //如果执行了use操作应该将当前Statement的catalog设置为新的database + boolean result = true; + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + if (SqlSyntaxValidator.isUseSql(sql)) { + HttpClientPoolUtil.execute(url, sql); + this.database = sql.trim().replace("use", "").trim(); + this.conn.setCatalog(this.database); + result = false; + } else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { + executeOneQuery(url, sql); + } else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { + executeOneUpdate(url, sql); + result = false; + } else { + if (SqlSyntaxValidator.isValidForExecuteQuery(sql)) { + executeQuery(sql); + } else { + executeUpdate(sql); + result = false; + } + } + return result; } - @Override - public void setCursorName(String name) throws SQLException { + private ResultSet executeOneQuery(String url, String sql) throws SQLException { + if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); - } - - @Override - public boolean execute(String sql) throws SQLException { - if (isClosed()) { - throw new SQLException("Invalid method call on a closed statement."); - } - //如果执行了use操作应该将当前Statement的catalog设置为新的database - if (SqlSyntaxValidator.isUseSql(sql)) { - this.database = sql.trim().replace("use", "").trim(); + // row data + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject resultJson = JSON.parseObject(result); + if (resultJson.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + resultJson.getString("desc") + "\n" + "error code: " + resultJson.getString("code"))); } - if (this.database == null) - throw new SQLException("Database not specified or available"); + // parse table name from sql +// String[] tableIdentifiers = parseTableIdentifier(sql); +// if (tableIdentifiers != null) { +// List fieldJsonList = new ArrayList<>(); +// for (String tableIdentifier : tableIdentifiers) { +// String fields = HttpClientPoolUtil.execute(url, "DESCRIBE " + tableIdentifier); +// JSONObject fieldJson = JSON.parseObject(fields); +// if (fieldJson.getString("status").equals("error")) { +// throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + fieldJson.getString("desc") + "\n" + "error code: " + fieldJson.getString("code"))); +// } +// fieldJsonList.add(fieldJson); +// } +// this.resultSet = new RestfulResultSet(database, this, resultJson, fieldJsonList); +// } else { + this.resultSet = new RestfulResultSet(database, this, resultJson); +// } + this.affectedRows = 0; + return resultSet; + } + + private int executeOneUpdate(String url, String sql) throws SQLException { + if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql); - final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - // use database - HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); - // execute sql String result = HttpClientPoolUtil.execute(url, sql); - // parse result JSONObject jsonObject = JSON.parseObject(result); if (jsonObject.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + - jsonObject.getString("desc") + "\n" + - "error code: " + jsonObject.getString("code"))); + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + jsonObject.getString("desc") + "\n" + "error code: " + jsonObject.getString("code"))); } - return true; + this.resultSet = null; + this.affectedRows = checkJsonResultSet(jsonObject); + return this.affectedRows; + } + + private int checkJsonResultSet(JSONObject jsonObject) { + // create ... SQLs should return 0 , and Restful result is this: + // {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1} + JSONArray head = jsonObject.getJSONArray("head"); + JSONArray data = jsonObject.getJSONArray("data"); + int rows = Integer.parseInt(jsonObject.getString("rows")); + if (head.size() == 1 && "affected_rows".equals(head.getString(0)) + && data.size() == 1 && data.getJSONArray(0).getInteger(0) == 0 && rows == 1) { + return 0; + } + return rows; } @Override public ResultSet getResultSet() throws SQLException { - return null; + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return resultSet; } @Override public int getUpdateCount() throws SQLException { - return 0; - } - - @Override - public boolean getMoreResults() throws SQLException { - return false; - } - - @Override - public void setFetchDirection(int direction) throws SQLException { - - } - - @Override - public int getFetchDirection() throws SQLException { - return 0; - } - - @Override - public void setFetchSize(int rows) throws SQLException { - - } - - @Override - public int getFetchSize() throws SQLException { - return 0; - } - - @Override - public int getResultSetConcurrency() throws SQLException { - return 0; - } - - @Override - public int getResultSetType() throws SQLException { - return 0; - } - - @Override - public void addBatch(String sql) throws SQLException { - - } - - @Override - public void clearBatch() throws SQLException { - - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - @Override - public int[] executeBatch() throws SQLException { - return new int[0]; + return this.affectedRows; } @Override public Connection getConnection() throws SQLException { - return null; - } - - @Override - public boolean getMoreResults(int current) throws SQLException { - return false; - } - - @Override - public ResultSet getGeneratedKeys() throws SQLException { - return null; - } - - @Override - public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { - return 0; - } - - @Override - public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { - return 0; - } - - @Override - public int executeUpdate(String sql, String[] columnNames) throws SQLException { - return 0; - } - - @Override - public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { - return false; - } - - @Override - public boolean execute(String sql, int[] columnIndexes) throws SQLException { - return false; - } - - @Override - public boolean execute(String sql, String[] columnNames) throws SQLException { - return false; - } - - @Override - public int getResultSetHoldability() throws SQLException { - return 0; + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return this.conn; } @Override @@ -293,37 +224,5 @@ public class RestfulStatement implements Statement { return closed; } - @Override - public void setPoolable(boolean poolable) throws SQLException { - - } - - @Override - public boolean isPoolable() throws SQLException { - return false; - } - - @Override - public void closeOnCompletion() throws SQLException { - - } - @Override - public boolean isCloseOnCompletion() throws SQLException { - return false; - } - - @Override - public T unwrap(Class iface) throws SQLException { - try { - return iface.cast(this); - } catch (ClassCastException cce) { - throw new SQLException("Unable to unwrap to " + iface.toString()); - } - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return iface.isInstance(this); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java similarity index 80% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java rename to src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java index 65399b122d97254b88b6bc2ef08910d7badc5061..5e2440fd1d0e551690c5f17fe60b30c1d4a377dc 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java @@ -1,6 +1,5 @@ -package com.taosdata.jdbc.rs.util; +package com.taosdata.jdbc.utils; -import org.apache.commons.lang3.StringUtils; import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; import org.apache.http.HttpEntity; @@ -17,10 +16,13 @@ import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; +import java.nio.charset.Charset; + public class HttpClientPoolUtil { public static PoolingHttpClientConnectionManager cm = null; public static CloseableHttpClient httpClient = null; + public static String token = "cm9vdDp0YW9zZGF0YQ=="; /** * 默认content 类型 */ @@ -36,7 +38,7 @@ public class HttpClientPoolUtil { /** * 初始化连接池 */ - public static synchronized void initPools() { + private static synchronized void initPools() { if (httpClient == null) { cm = new PoolingHttpClientConnectionManager(); cm.setDefaultMaxPerRoute(count); @@ -48,7 +50,7 @@ public class HttpClientPoolUtil { /** * Http connection keepAlive 设置 */ - public static ConnectionKeepAliveStrategy defaultStrategy = (response, context) -> { + private static ConnectionKeepAliveStrategy defaultStrategy = (response, context) -> { HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); int keepTime = Http_Default_Keep_Time * 1000; while (it.hasNext()) { @@ -59,23 +61,13 @@ public class HttpClientPoolUtil { try { return Long.parseLong(value) * 1000; } catch (Exception e) { - new Exception( - "format KeepAlive timeout exception, exception:" + e.toString()) - .printStackTrace(); + new Exception("format KeepAlive timeout exception, exception:" + e.toString()).printStackTrace(); } } } return keepTime; }; - public static CloseableHttpClient getHttpClient() { - return httpClient; - } - - public static PoolingHttpClientConnectionManager getHttpConnectionManager() { - return cm; - } - /** * 执行http post请求 * 默认采用Content-Type：application/json，Accept：application/json @@ -94,7 +86,11 @@ public class HttpClientPoolUtil { initPools(); } method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME, DEFAULT_CONTENT_TYPE, 0); - method.setEntity(new StringEntity(data)); + method.setHeader("Content-Type", "text/plain"); + method.setHeader("Connection", "keep-alive"); + method.setHeader("Authorization", "Taosd " + token); + + method.setEntity(new StringEntity(data, Charset.forName("UTF-8"))); HttpContext context = HttpClientContext.create(); CloseableHttpResponse httpResponse = httpClient.execute(method, context); httpEntity = httpResponse.getEntity(); @@ -105,26 +101,13 @@ public class HttpClientPoolUtil { if (method != null) { method.abort(); } -// e.printStackTrace(); -// logger.error("execute post request exception, url:" + uri + ", exception:" + e.toString() -// + ", cost time(ms):" + (System.currentTimeMillis() - startTime)); - new Exception("execute post request exception, url:" - + uri + ", exception:" + e.toString() + - ", cost time(ms):" + (System.currentTimeMillis() - startTime)) - .printStackTrace(); + new Exception("execute post request exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):" + (System.currentTimeMillis() - startTime)).printStackTrace(); } finally { if (httpEntity != null) { try { EntityUtils.consumeQuietly(httpEntity); } catch (Exception e) { -// e.printStackTrace(); -// logger.error("close response exception, url:" + uri + ", exception:" + e.toString() -// + ", cost time(ms):" + (System.currentTimeMillis() - startTime)); - new Exception( - "close response exception, url:" + uri + - ", exception:" + e.toString() - + ", cost time(ms):" + (System.currentTimeMillis() - startTime)) - .printStackTrace(); + new Exception("close response exception, url:" + uri + ", exception:" + e.toString() + ", cost time(ms):" + (System.currentTimeMillis() - startTime)).printStackTrace(); } } } @@ -141,7 +124,7 @@ public class HttpClientPoolUtil { * @return HttpRequestBase 返回类型 * @author lisc */ - public static HttpRequestBase getRequest(String uri, String methodName, String contentType, int timeout) { + private static HttpRequestBase getRequest(String uri, String methodName, String contentType, int timeout) { if (httpClient == null) { initPools(); } @@ -162,7 +145,7 @@ public class HttpClientPoolUtil { method = new HttpPost(uri); } - if (StringUtils.isBlank(contentType)) { + if (contentType == null || contentType.isEmpty() || contentType.replaceAll("\\s", "").isEmpty()) { contentType = DEFAULT_CONTENT_TYPE; } method.addHeader("Content-Type", contentType); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index 388c3978bef3372e6abca89a70bd428a8bb79d3d..251ca2af013e2b1c9cb314b776621455c91d9384 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -15,15 +15,16 @@ package com.taosdata.jdbc.utils; import com.taosdata.jdbc.TSDBConnection; -import com.taosdata.jdbc.TSDBJNIConnector; import java.sql.Connection; -import java.sql.SQLException; public class SqlSyntaxValidator { - private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use"}; - private static final String[] querySQL = {"select"}; + private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe"}; + private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"}; + private static final String[] querySQL = {"select", "show", "describe"}; + + private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"}; private TSDBConnection tsdbConnection; @@ -31,43 +32,68 @@ public class SqlSyntaxValidator { this.tsdbConnection = (TSDBConnection) connection; } - public boolean validateSqlSyntax(String sql) throws SQLException { - - boolean res = false; - if (tsdbConnection == null || tsdbConnection.isClosed()) { - throw new SQLException("invalid connection"); - } else { - TSDBJNIConnector jniConnector = tsdbConnection.getConnection(); - if (jniConnector == null) { - throw new SQLException("jniConnector is null"); - } else { - res = jniConnector.validateCreateTableSql(sql); - } + public static boolean isValidForExecuteUpdate(String sql) { + for (String prefix : updateSQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; } - return res; + return false; } - public static boolean isValidForExecuteUpdate(String sql) { - for (String prefix : updateSQL) { + public static boolean isValidForExecuteQuery(String sql) { + for (String prefix : querySQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isValidForExecute(String sql) { + for (String prefix : SQL) { if (sql.trim().toLowerCase().startsWith(prefix)) return true; } return false; } + public static boolean isDatabaseUnspecifiedQuery(String sql) { + for (String databaseObj : databaseUnspecifiedShow) { + if (sql.trim().toLowerCase().matches("show\\s+" + databaseObj + ".*")) + return true; + } + return false; + } + + public static boolean isDatabaseUnspecifiedUpdate(String sql) { + sql = sql.trim().toLowerCase(); + return sql.matches("create\\s+database.*") || sql.startsWith("set") || sql.matches("drop\\s+database.*"); + } + public static boolean isUseSql(String sql) { - return sql.trim().toLowerCase().startsWith(updateSQL[8]) || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); + return sql.trim().toLowerCase().startsWith("use"); +// || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); } - public static boolean isUpdateSql(String sql) { - return sql.trim().toLowerCase().startsWith(updateSQL[1]); + public static boolean isShowSql(String sql) { + return sql.trim().toLowerCase().startsWith("show"); } + public static boolean isDescribeSql(String sql) { + return sql.trim().toLowerCase().startsWith("describe"); + } + + public static boolean isInsertSql(String sql) { - return sql.trim().toLowerCase().startsWith(updateSQL[0]); + return sql.trim().toLowerCase().startsWith("insert") || sql.trim().toLowerCase().startsWith("import"); } public static boolean isSelectSql(String sql) { - return sql.trim().toLowerCase().startsWith(querySQL[0]); + return sql.trim().toLowerCase().startsWith("select"); } + + public static boolean isShowDatabaseSql(String sql) { + return sql.trim().toLowerCase().matches("show\\s*databases"); + } + + } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java deleted file mode 100644 index 800265868da6da5047453d542fc9e37c69fe1e36..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java +++ /dev/null @@ -1,272 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.util.*; -import java.util.concurrent.TimeUnit; - -public class TDNode { - - private int index; - private int running; - private int deployed; - private boolean testCluster; - private String path; - private String cfgDir; - private String dataDir; - private String logDir; - private String cfgPath; - - public TDNode(int index) { - this.index = index; - running = 0; - deployed = 0; - testCluster = false; - } - - public void setPath(String path) { - this.path = path; - } - - public void setTestCluster(boolean testCluster) { - this.testCluster = testCluster; - } - - public void setRunning(int running) { - this.running = running; - } - - public void searchTaosd(File dir, ArrayList taosdPath) { - File[] fileList = dir.listFiles(); - - if(fileList == null || fileList.length == 0) { - return; - } - - for(File file : fileList) { - if(file.isFile()) { - if(file.getName().equals("taosd")) { - taosdPath.add(file.getAbsolutePath()); - } - } else { - searchTaosd(file, taosdPath); - } - } - } - - public void start() { - String selfPath = System.getProperty("user.dir"); - String binPath = ""; - String projDir = selfPath + "/../../../"; - - try { - ArrayList taosdPath = new ArrayList<>(); - - File dir = new File(projDir); - String realProjDir = dir.getCanonicalPath(); - dir = new File(realProjDir); - System.out.println("project Dir: " + projDir); - searchTaosd(dir, taosdPath); - - if(taosdPath.size() == 0) { - System.out.println("The project path doens't exist"); - return; - } else { - for(String p : taosdPath) { - if(!p.contains("packaging")) { - binPath = p; - } - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - if(binPath.isEmpty()) { - System.out.println("taosd not found"); - return; - } else { - System.out.println("taosd found in " + binPath); - } - - if(this.deployed == 0) { - System.out.println("dnode" + index + "is not deployed"); - return; - } - - String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & "; - System.out.println("start taosd cmd: " + cmd); - - try{ - Runtime.getRuntime().exec(cmd); - TimeUnit.SECONDS.sleep(5); - } catch (Exception e) { - e.printStackTrace(); - } - - this.running = 1; - } - - public Integer getTaosdPid() { - String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"; - String[] cmds = {"sh", "-c", cmd}; - try { - Process process = Runtime.getRuntime().exec(cmds); - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line = null; - Integer res = null; - while((line = reader.readLine()) != null) { - if(!line.isEmpty()) { - res = Integer.valueOf(line); - break; - } - } - - return res; - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public void stop() { - - if (this.running != 0) { - Integer pid = null; - while((pid = getTaosdPid()) != null) { - - String killCmd = "kill -term " + pid; - String[] killCmds = {"sh", "-c", killCmd}; - try { - Runtime.getRuntime().exec(killCmds).waitFor(); - - TimeUnit.SECONDS.sleep(2); - } catch (Exception e) { - e.printStackTrace(); - } - } - - try { - for(int port = 6030; port < 6041; port ++) { - String fuserCmd = "fuser -k -n tcp " + port; - Runtime.getRuntime().exec(fuserCmd).waitFor(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - this.running = 0; - System.out.println("dnode:" + this.index + " is stopped by kill -term"); - } - } - - public void startIP() { - try{ - String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up"; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void stopIP() { - try{ - String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down"; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void setCfgConfig(String option, String value) { - try{ - String cmd = "echo " + option + " " + value + " >> " + this.cfgPath; - String[] cmdLine = {"sh", "-c", cmd}; - Process ps = Runtime.getRuntime().exec(cmdLine); - ps.waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public String getDnodeRootDir() { - String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index; - return dnodeRootDir; - } - - public String getDnodesRootDir() { - String dnodesRootDir = this.path + "/sim/psim" + this.index; - return dnodesRootDir; - } - - public void deploy() { - this.logDir = this.path + "/sim/dnode" + this.index + "/log"; - this.dataDir = this.path + "/sim/dnode" + this.index + "/data"; - this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg"; - this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg"; - - try { - String cmd = "rm -rf " + this.logDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "rm -rf " + this.cfgDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "rm -rf " + this.dataDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.logDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.cfgDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.dataDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "touch " + this.cfgPath; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - - if(this.testCluster) { - startIP(); - setCfgConfig("masterIp", "192.168.0.1"); - setCfgConfig("secondIp", "192.168.0.2"); - setCfgConfig("publicIp", "192.168.0." + this.index); - setCfgConfig("internalIp", "192.168.0." + this.index); - setCfgConfig("privateIp", "192.168.0." + this.index); - } - setCfgConfig("dataDir", this.dataDir); - setCfgConfig("logDir", this.logDir); - setCfgConfig("numOfLogLines", "1000000/00"); - setCfgConfig("mnodeEqualVnodeNum", "0"); - setCfgConfig("walLevel", "1"); - setCfgConfig("statusInterval", "1"); - setCfgConfig("numOfMnodes", "3"); - setCfgConfig("numOfThreadsPerCore", "2.0"); - setCfgConfig("monitor", "0"); - setCfgConfig("maxVnodeConnections", "30000"); - setCfgConfig("maxMgmtConnections", "30000"); - setCfgConfig("maxMeterConnections", "30000"); - setCfgConfig("maxShellConns", "30000"); - setCfgConfig("locale", "en_US.UTF-8"); - setCfgConfig("charset", "UTF-8"); - setCfgConfig("asyncLog", "0"); - setCfgConfig("anyIp", "0"); - setCfgConfig("dDebugFlag", "135"); - setCfgConfig("mDebugFlag", "135"); - setCfgConfig("sdbDebugFlag", "135"); - setCfgConfig("rpcDebugFlag", "135"); - setCfgConfig("tmrDebugFlag", "131"); - setCfgConfig("cDebugFlag", "135"); - setCfgConfig("httpDebugFlag", "135"); - setCfgConfig("monitorDebugFlag", "135"); - setCfgConfig("udebugFlag", "135"); - setCfgConfig("jnidebugFlag", "135"); - setCfgConfig("qdebugFlag", "135"); - this.deployed = 1; - } -} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java deleted file mode 100644 index efc4c53e2864789811ca30110f9b69e5f685a89e..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.io.File; -import java.util.*; - -public class TDNodes { - private ArrayList tdNodes; - private boolean testCluster; - - public TDNodes () { - tdNodes = new ArrayList<>(); - for(int i = 1; i < 11; i ++) { - tdNodes.add(new TDNode(i)); - } - } - - public void setTestCluster(boolean testCluster) { - this.testCluster = testCluster; - } - - public void check(int index) { - if(index < 1 || index > 10) { - System.out.println("index: " + index + " should on a scale of [1, 10]"); - return; - } - } - - public void deploy(int index) { - try { - File file = new File(System.getProperty("user.dir") + "/../../../"); - String projectRealPath = file.getCanonicalPath(); - check(index); - tdNodes.get(index - 1).setTestCluster(this.testCluster); - tdNodes.get(index - 1).setPath(projectRealPath); - tdNodes.get(index - 1).deploy(); - } catch (Exception e) { - e.printStackTrace(); - System.out.println("deploy Test Exception"); - } - } - - public void cfg(int index, String option, String value) { - check(index); - tdNodes.get(index - 1).setCfgConfig(option, value); - } - - public TDNode getTDNode(int index) { - check(index); - return tdNodes.get(index - 1); - } - - public void start(int index) { - check(index); - tdNodes.get(index - 1).start(); - } - - public void stop(int index) { - check(index); - tdNodes.get(index - 1).stop(); - } - - public void startIP(int index) { - check(index); - tdNodes.get(index - 1).startIP(); - } - - public void stopIP(int index) { - check(index); - tdNodes.get(index - 1).stopIP(); - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..ee1364ce217c639cbf864f87f83ad6f5ddd6c137 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java @@ -0,0 +1,74 @@ +package com.taosdata.jdbc.utils; + +import javax.management.*; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicLong; + +public class TaosInfo implements TaosInfoMBean { + + private static volatile TaosInfo instance; + private AtomicLong connect_open = new AtomicLong(); + private AtomicLong connect_close = new AtomicLong(); + private AtomicLong statement_count = new AtomicLong(); + + static { + try { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = new ObjectName("TaosInfoMBean:name=TaosInfo"); + server.registerMBean(TaosInfo.getInstance(), name); + + } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) { + e.printStackTrace(); + } + } + + @Override + public long getConnect_open() { + return connect_open.get(); + } + + @Override + public long getConnect_close() { + return connect_close.get(); + } + + @Override + public long getConnect_active() { + return connect_open.get() - connect_close.get(); + } + + @Override + public long getStatement_count() { + return statement_count.get(); + } + + /*******************************************************/ + + public void conn_open_increment() { + connect_open.incrementAndGet(); + } + + public void connect_close_increment() { + connect_close.incrementAndGet(); + } + + public void stmt_count_increment() { + statement_count.incrementAndGet(); + } + + /********************************************************************************/ + private TaosInfo() { + } + + public static TaosInfo getInstance() { + if (instance == null) { + synchronized (TaosInfo.class) { + if (instance == null) { + instance = new TaosInfo(); + } + } + } + return instance; + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java new file mode 100644 index 0000000000000000000000000000000000000000..e16f41b2f585244721e923a963c53fb6b7bccf47 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java @@ -0,0 +1,13 @@ +package com.taosdata.jdbc.utils; + +public interface TaosInfoMBean { + + long getConnect_open(); + + long getConnect_close(); + + long getConnect_active(); + + long getStatement_count(); + +} diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java deleted file mode 100644 index 1edb957493183c407b668bd68532e83fa1a64dba..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java +++ /dev/null @@ -1,36 +0,0 @@ -import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.TSDBPreparedStatement; - -import java.sql.*; -import java.util.Properties; - -public class TestPreparedStatement { - - public static void main(String[] args) throws SQLException { - Connection connection = null; - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); - String rawSql = "select * from test.log0601"; -// String[] params = new String[]{"ts", "c1"}; - PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); - ResultSet resSet = pstmt.executeQuery(); - while(resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s \n", i, resSet.getString(i)); - } - } - resSet.close(); - pstmt.close(); - connection.close(); - - } catch (Exception e) { - e.printStackTrace(); - if (null != connection) { - connection.close(); - } - } - } -} diff --git a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java deleted file mode 100644 index 39a08f0fe98a279e62ca07f20047813edeab56db..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java +++ /dev/null @@ -1,33 +0,0 @@ -import com.taosdata.jdbc.TSDBDriver; - -import java.sql.*; -import java.util.Properties; - -public class TestTSDBDatabaseMetaData { - - public static void main(String[] args) throws SQLException { - Connection connection = null; - DatabaseMetaData dbMetaData = null; - ResultSet resSet = null; - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); - dbMetaData = connection.getMetaData(); - resSet = dbMetaData.getCatalogs(); - while(resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf("dbMetaData.getCatalogs(%d) = %s\n", i, resSet.getString(i)); - } - } - resSet.close(); - - } catch (Exception e) { - e.printStackTrace(); - if (null != connection) { - connection.close(); - } - } - } -} diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java deleted file mode 100644 index 47acb200649be4c8c6af5817c2ef4c074a2a3fa0..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java +++ /dev/null @@ -1,78 +0,0 @@ -import com.taosdata.jdbc.TSDBConnection; -import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.TSDBResultSet; -import com.taosdata.jdbc.TSDBSubscribe; - -import java.sql.DriverManager; -import java.util.Properties; - -public class TestTSDBSubscribe { - - public static TSDBConnection connectTDengine(String host, String database) throws Exception { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - - String cs = String.format("jdbc:TAOS://%s:0/%s", host, database); - return (TSDBConnection)DriverManager.getConnection(cs, properties); - } - - public static void main(String[] args) throws Exception { - String usage = "java -Djava.ext.dirs=../ TestTSDBSubscribe [-host host] <-db database> <-topic topic> <-sql sql>"; - if (args.length < 2) { - System.err.println(usage); - return; - } - - String host = "localhost", database = "", topic = "", sql = ""; - for (int i = 0; i < args.length; i++) { - if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { - database = args[++i]; - } - if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { - topic = args[++i]; - } - if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { - host = args[++i]; - } - if ("-sql".equalsIgnoreCase(args[i]) && i < args.length - 1) { - sql = args[++i]; - } - } - if (database.isEmpty() || topic.isEmpty() || sql.isEmpty()) { - System.err.println(usage); - return; - } - - TSDBConnection connection = null; - TSDBSubscribe sub = null; - try { - connection = connectTDengine(host, database); - sub = ((TSDBConnection) connection).subscribe(topic, sql, false); - - int total = 0; - while(true) { - TSDBResultSet rs = sub.consume(); - int count = 0; - while(rs.next()) { - count++; - } - total += count; - System.out.printf("%d rows consumed, total %d\n", count, total); - Thread.sleep(900); - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (null != sub) { - sub.close(true); - } - if (null != connection) { - connection.close(); - } - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java deleted file mode 100644 index ce3735c12894807efadd1f5673fc34eee43ae01b..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.taosdata.jdbc; - -import com.taosdata.jdbc.utils.TDNodes; - -import org.junit.AfterClass; -import org.junit.BeforeClass; - -public abstract class BaseTest { - - private static boolean testCluster = false; - private static TDNodes nodes = new TDNodes(); - - @BeforeClass - public static void setupEnv() { - try { - if (nodes.getTDNode(1).getTaosdPid() != null) { - System.out.println("Kill taosd before running JDBC test"); - nodes.getTDNode(1).setRunning(1); - nodes.stop(1); - } - nodes.setTestCluster(testCluster); - nodes.deploy(1); - nodes.start(1); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @AfterClass - public static void cleanUpEnv() { - nodes.stop(1); - } -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java deleted file mode 100644 index 7d96cbb5381d099f7b67ecdb5414093c54b37b65..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; - -public class BatchInsertTest extends BaseTest { - - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "meters"; - static String host = "localhost"; - static int numOfTables = 30; - final static int numOfRecordsPerTable = 1000; - static long ts = 1496732686000l; - final static String tablePrefix = "t"; - - @Before - public void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - - String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; - statement.executeUpdate(createTableSql); - - for(int i = 0; i < numOfTables; i++) { - String loc = i % 2 == 0 ? "beijing" : "shanghai"; - String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; - statement.executeUpdate(createSubTalbesSql); - } - - } - - @Test - public void testBatchInsert() throws SQLException{ - - ExecutorService executorService = Executors.newFixedThreadPool(numOfTables); - - for (int i = 0; i < numOfTables; i++) { - final int index = i; - executorService.execute(new Runnable() { - @Override - public void run() { - try { - long startTime = System.currentTimeMillis(); - Statement statement = connection.createStatement(); // get statement - StringBuilder sb = new StringBuilder(); - sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); - Random rand = new Random(); - for (int j = 1; j <= numOfRecordsPerTable; j++) { - sb.append("(" + (ts + j) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ")"); - } - statement.addBatch(sb.toString()); - statement.executeBatch(); - long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); - connection.commit(); - statement.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - }); - } - - executorService.shutdown(); - - try { - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - Statement statement = connection.createStatement(); - ResultSet rs = statement.executeQuery("select * from meters"); - int num = 0; - while (rs.next()) { - num++; - } - assertEquals(num, numOfTables * numOfRecordsPerTable); - rs.close(); - } - - - @After - public void close() throws Exception { - statement.close(); - connection.close(); - Thread.sleep(10); - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java deleted file mode 100644 index a54ece4eadc54e930a4af4faddb58dfef3ceb25a..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.Test; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; - -import static org.junit.Assert.assertTrue; - -public class ConnectionTest extends BaseTest { - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "st"; - static String host = "localhost"; - - @Test - public void testConnection() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - assertTrue(null != connection); - statement = connection.createStatement(); - assertTrue(null != statement); - - // try reconnect - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - try { - statement.execute("create database if not exists " + dbName); - } catch (SQLException e) { - assert false : "create database error: " + e.getMessage(); - } - - try { - if (!connection.isClosed()) { - if (!statement.isClosed()) { - statement.executeUpdate("drop database " + dbName); - statement.close(); - } - connection.close(); - Thread.sleep(10); - } - } catch (Exception e) { - assert false : "close connection error: " + e.getMessage(); - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java deleted file mode 100644 index 284af3dfe75f7c436ec2ce875714afc235c525d8..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java +++ /dev/null @@ -1,244 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; - -public class DatabaseMetaDataTest extends BaseTest { - static Connection connection = null; - static PreparedStatement statement = null; - static String dbName = "test"; - static String tName = "t0"; - static String host = "localhost"; - - @BeforeClass - public static void createConnection() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - String sql = "drop database if exists " + dbName; - statement = (TSDBPreparedStatement) connection.prepareStatement(sql); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - } - - @Test - public void testMetaDataTest() throws SQLException { - - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet resultSet = databaseMetaData.getTables(dbName, "t*", "t*", new String[]{"t"}); - while (resultSet.next()) { - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s\n", i, resultSet.getString(i)); - } - } - resultSet.close(); - databaseMetaData.isWrapperFor(null); - databaseMetaData.allProceduresAreCallable(); - databaseMetaData.allTablesAreSelectable(); - databaseMetaData.getURL(); - databaseMetaData.getUserName(); - databaseMetaData.isReadOnly(); - databaseMetaData.nullsAreSortedHigh(); - databaseMetaData.nullsAreSortedLow(); - databaseMetaData.nullsAreSortedAtStart(); - databaseMetaData.nullsAreSortedAtEnd(); - databaseMetaData.getDatabaseProductName(); - databaseMetaData.getDatabaseProductVersion(); - databaseMetaData.getDriverName(); - databaseMetaData.getDriverVersion(); - databaseMetaData.getDriverMajorVersion(); - databaseMetaData.getDriverMinorVersion(); - databaseMetaData.usesLocalFiles(); - databaseMetaData.usesLocalFilePerTable(); - databaseMetaData.supportsMixedCaseIdentifiers(); - databaseMetaData.storesUpperCaseIdentifiers(); - databaseMetaData.storesLowerCaseIdentifiers(); - databaseMetaData.storesMixedCaseIdentifiers(); - databaseMetaData.supportsMixedCaseQuotedIdentifiers(); - databaseMetaData.storesUpperCaseQuotedIdentifiers(); - databaseMetaData.storesLowerCaseQuotedIdentifiers(); - databaseMetaData.storesMixedCaseQuotedIdentifiers(); - databaseMetaData.getIdentifierQuoteString(); - databaseMetaData.getSQLKeywords(); - databaseMetaData.getNumericFunctions(); - databaseMetaData.getStringFunctions(); - databaseMetaData.getSystemFunctions(); - databaseMetaData.getTimeDateFunctions(); - databaseMetaData.getSearchStringEscape(); - databaseMetaData.getExtraNameCharacters(); - databaseMetaData.supportsAlterTableWithAddColumn(); - databaseMetaData.supportsAlterTableWithDropColumn(); - databaseMetaData.supportsColumnAliasing(); - databaseMetaData.nullPlusNonNullIsNull(); - databaseMetaData.supportsConvert(); - databaseMetaData.supportsConvert(0, 0); - databaseMetaData.supportsTableCorrelationNames(); - databaseMetaData.supportsDifferentTableCorrelationNames(); - databaseMetaData.supportsExpressionsInOrderBy(); - databaseMetaData.supportsOrderByUnrelated(); - databaseMetaData.supportsGroupBy(); - databaseMetaData.supportsGroupByUnrelated(); - databaseMetaData.supportsGroupByBeyondSelect(); - databaseMetaData.supportsLikeEscapeClause(); - databaseMetaData.supportsMultipleResultSets(); - databaseMetaData.supportsMultipleTransactions(); - databaseMetaData.supportsNonNullableColumns(); - databaseMetaData.supportsMinimumSQLGrammar(); - databaseMetaData.supportsCoreSQLGrammar(); - databaseMetaData.supportsExtendedSQLGrammar(); - databaseMetaData.supportsANSI92EntryLevelSQL(); - databaseMetaData.supportsANSI92IntermediateSQL(); - databaseMetaData.supportsANSI92FullSQL(); - databaseMetaData.supportsIntegrityEnhancementFacility(); - databaseMetaData.supportsOuterJoins(); - databaseMetaData.supportsFullOuterJoins(); - databaseMetaData.supportsLimitedOuterJoins(); - databaseMetaData.getSchemaTerm(); - databaseMetaData.getProcedureTerm(); - databaseMetaData.getCatalogTerm(); - databaseMetaData.isCatalogAtStart(); - databaseMetaData.getCatalogSeparator(); - databaseMetaData.supportsSchemasInDataManipulation(); - databaseMetaData.supportsSchemasInProcedureCalls(); - databaseMetaData.supportsSchemasInTableDefinitions(); - databaseMetaData.supportsSchemasInIndexDefinitions(); - databaseMetaData.supportsSchemasInPrivilegeDefinitions(); - databaseMetaData.supportsCatalogsInDataManipulation(); - databaseMetaData.supportsCatalogsInProcedureCalls(); - databaseMetaData.supportsCatalogsInTableDefinitions(); - databaseMetaData.supportsCatalogsInIndexDefinitions(); - databaseMetaData.supportsCatalogsInPrivilegeDefinitions(); - databaseMetaData.supportsPositionedDelete(); - databaseMetaData.supportsPositionedUpdate(); - databaseMetaData.supportsSelectForUpdate(); - databaseMetaData.supportsStoredProcedures(); - databaseMetaData.supportsSubqueriesInComparisons(); - databaseMetaData.supportsSubqueriesInExists(); - databaseMetaData.supportsSubqueriesInIns(); - databaseMetaData.supportsSubqueriesInQuantifieds(); - databaseMetaData.supportsCorrelatedSubqueries(); - databaseMetaData.supportsUnion(); - databaseMetaData.supportsUnionAll(); - databaseMetaData.supportsOpenCursorsAcrossCommit(); - databaseMetaData.supportsOpenCursorsAcrossRollback(); - databaseMetaData.supportsOpenStatementsAcrossCommit(); - databaseMetaData.supportsOpenStatementsAcrossRollback(); - databaseMetaData.getMaxBinaryLiteralLength(); - databaseMetaData.getMaxCharLiteralLength(); - databaseMetaData.getMaxColumnNameLength(); - databaseMetaData.getMaxColumnsInGroupBy(); - databaseMetaData.getMaxColumnsInIndex(); - databaseMetaData.getMaxColumnsInOrderBy(); - databaseMetaData.getMaxColumnsInSelect(); - databaseMetaData.getMaxColumnsInTable(); - databaseMetaData.getMaxConnections(); - databaseMetaData.getMaxCursorNameLength(); - databaseMetaData.getMaxIndexLength(); - databaseMetaData.getMaxSchemaNameLength(); - databaseMetaData.getMaxProcedureNameLength(); - databaseMetaData.getMaxCatalogNameLength(); - databaseMetaData.getMaxRowSize(); - databaseMetaData.doesMaxRowSizeIncludeBlobs(); - databaseMetaData.getMaxStatementLength(); - databaseMetaData.getMaxStatements(); - databaseMetaData.getMaxTableNameLength(); - databaseMetaData.getMaxTablesInSelect(); - databaseMetaData.getMaxUserNameLength(); - databaseMetaData.getDefaultTransactionIsolation(); - databaseMetaData.supportsTransactions(); - databaseMetaData.supportsTransactionIsolationLevel(0); - databaseMetaData.supportsDataDefinitionAndDataManipulationTransactions(); - databaseMetaData.supportsDataManipulationTransactionsOnly(); - databaseMetaData.dataDefinitionCausesTransactionCommit(); - databaseMetaData.dataDefinitionIgnoredInTransactions(); - try { - databaseMetaData.getProcedures("", "", ""); - } catch (Exception e) { - } - try { - databaseMetaData.getProcedureColumns("", "", "", ""); - } catch (Exception e) { - } - try { - databaseMetaData.getTables("", "", "", new String[]{""}); - } catch (Exception e) { - } - databaseMetaData.getSchemas(); - databaseMetaData.getCatalogs(); -// databaseMetaData.getTableTypes(); - - databaseMetaData.getColumns("", "", "", ""); - databaseMetaData.getColumnPrivileges("", "", "", ""); - databaseMetaData.getTablePrivileges("", "", ""); - databaseMetaData.getBestRowIdentifier("", "", "", 0, false); - databaseMetaData.getVersionColumns("", "", ""); - databaseMetaData.getPrimaryKeys("", "", ""); - databaseMetaData.getImportedKeys("", "", ""); - databaseMetaData.getExportedKeys("", "", ""); - databaseMetaData.getCrossReference("", "", "", "", "", ""); - databaseMetaData.getTypeInfo(); - databaseMetaData.getIndexInfo("", "", "", false, false); - databaseMetaData.supportsResultSetType(0); - databaseMetaData.supportsResultSetConcurrency(0, 0); - databaseMetaData.ownUpdatesAreVisible(0); - databaseMetaData.ownDeletesAreVisible(0); - databaseMetaData.ownInsertsAreVisible(0); - databaseMetaData.othersUpdatesAreVisible(0); - databaseMetaData.othersDeletesAreVisible(0); - databaseMetaData.othersInsertsAreVisible(0); - databaseMetaData.updatesAreDetected(0); - databaseMetaData.deletesAreDetected(0); - databaseMetaData.insertsAreDetected(0); - databaseMetaData.supportsBatchUpdates(); - databaseMetaData.getUDTs("", "", "", new int[]{0}); - databaseMetaData.getConnection(); - databaseMetaData.supportsSavepoints(); - databaseMetaData.supportsNamedParameters(); - databaseMetaData.supportsMultipleOpenResults(); - databaseMetaData.supportsGetGeneratedKeys(); - databaseMetaData.getSuperTypes("", "", ""); - databaseMetaData.getSuperTables("", "", ""); - databaseMetaData.getAttributes("", "", "", ""); - databaseMetaData.supportsResultSetHoldability(0); - databaseMetaData.getResultSetHoldability(); - databaseMetaData.getDatabaseMajorVersion(); - databaseMetaData.getDatabaseMinorVersion(); - databaseMetaData.getJDBCMajorVersion(); - databaseMetaData.getJDBCMinorVersion(); - databaseMetaData.getSQLStateType(); - databaseMetaData.locatorsUpdateCopy(); - databaseMetaData.supportsStatementPooling(); - databaseMetaData.getRowIdLifetime(); - databaseMetaData.getSchemas("", ""); - databaseMetaData.supportsStoredFunctionsUsingCallSyntax(); - databaseMetaData.autoCommitFailureClosesAllResultSets(); - databaseMetaData.getClientInfoProperties(); - databaseMetaData.getFunctions("", "", ""); - databaseMetaData.getFunctionColumns("", "", "", ""); - databaseMetaData.getPseudoColumns("", "", "", ""); - databaseMetaData.generatedKeyAlwaysReturned(); - - } - - @AfterClass - public static void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java deleted file mode 100644 index dbe16d9fea33be117cfa93a3ecd9a1ff4bd12cd8..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; - -public class ImportTest extends BaseTest { - Connection connection = null; - Statement statement = null; - String dbName = "test"; - String tName = "t0"; - String host = "localhost"; - - @Before - public void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - - } - - @Test - public void insertData() throws Exception { - long ts = 1496732686000l; - - for (int i = 0; i < 50; i++) { - ts++; - int row = statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); - System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); - assertEquals(1, row); - } - } - - @Test - public void selectData() throws Exception { - insertData(); - String sql = "select * from test.t0"; - ResultSet resSet = statement.executeQuery(sql); - - while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); - } - } - resSet.close(); - } - - @Test - public void importData() throws Exception { - // 避免时间重复 - long ts = 1496732686000l; - - StringBuilder sqlBuilder = new StringBuilder("insert into ").append(dbName).append(".").append(tName).append(" values "); - - for (int i = 0; i < 50; i++) { - int a = i / 5; - long t = ts + a; - sqlBuilder.append("(").append(t).append(",").append((100 + i)).append(",").append(i).append(") "); - } - System.out.println(sqlBuilder.toString()); - int rows = statement.executeUpdate(sqlBuilder.toString()); - System.out.println(rows); - assertEquals(10, rows); - } - - @After - public void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java deleted file mode 100644 index 0535214ac198685ad65edf338bc6d72f841731a8..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java +++ /dev/null @@ -1,200 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@FixMethodOrder(MethodSorters.DEFAULT) -public class PreparedStatementTest extends BaseTest { - static Connection connection = null; - static PreparedStatement statement = null; - static String dbName = "test"; - static String tName = "t0"; - static String host = "localhost"; - static ResultSet resSet = null; - - @BeforeClass - public static void createConnection() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - String sql = "drop database if exists " + dbName; - statement = (TSDBPreparedStatement) connection.prepareStatement(sql); - - } - - @Test - public void createTableAndQuery() throws SQLException { - long ts = System.currentTimeMillis(); - - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + "(ts timestamp, k1 int)"); - statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); - - PreparedStatement selectStatement = connection.prepareStatement("select * from " + dbName + "." + tName); - - ResultSet resultSet = selectStatement.executeQuery(); - assertTrue(null != resultSet); - - boolean isClosed = statement.isClosed(); - assertEquals(false, isClosed); - } - - @Test - public void testPreparedStatement() throws SQLException { - long ts = System.currentTimeMillis() + 20000; - PreparedStatement saveStatement = connection - .prepareStatement("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); - - int affectedRows = saveStatement.executeUpdate(); - assertTrue(1 == affectedRows); - } - - @Test - public void testSavedPreparedStatement() throws SQLException { - long ts = System.currentTimeMillis(); - - TSDBPreparedStatement saveStatement = (TSDBPreparedStatement) connection - .prepareStatement("insert into " + dbName + "." + tName + " values (?, ?)"); - - saveStatement.setObject(1, ts + 10000); - saveStatement.setObject(2, 3); - int rows = saveStatement.executeUpdate(); - assertEquals(1, rows); - } - - @Test - public void testUnsupport() { - // if(null == resSet) { - // return; - // } - TSDBPreparedStatement tsdbStatement = (TSDBPreparedStatement) statement; - try { - tsdbStatement.unwrap(null); - } catch (SQLException e) { - } - try { - tsdbStatement.isWrapperFor(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMaxFieldSize(); - } catch (SQLException e) { - } - try { - tsdbStatement.setMaxFieldSize(0); - } catch (SQLException e) { - } - try { - tsdbStatement.setEscapeProcessing(true); - } catch (SQLException e) { - } - try { - tsdbStatement.cancel(); - } catch (SQLException e) { - } - try { - tsdbStatement.getWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.clearWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.setCursorName(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.setFetchDirection(0); - } catch (SQLException e) { - } - try { - tsdbStatement.getFetchDirection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetConcurrency(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetType(); - } catch (SQLException e) { - } - try { - tsdbStatement.getConnection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.getGeneratedKeys(); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, 0); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new int[]{0}); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"}); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetHoldability(); - } catch (SQLException e) { - } - try { - tsdbStatement.setPoolable(true); - } catch (SQLException e) { - } - try { - tsdbStatement.isPoolable(); - } catch (SQLException e) { - } - try { - tsdbStatement.closeOnCompletion(); - } catch (SQLException e) { - } - try { - tsdbStatement.isCloseOnCompletion(); - } catch (SQLException e) { - } - } - - @AfterClass - public static void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } - -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java deleted file mode 100644 index 611e21e887b1988a66019d437133d6c3f7926961..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; -import java.util.Random; - -import static org.junit.Assert.assertEquals; -import java.util.Properties; -import java.util.concurrent.Executors; -import java.util.concurrent.*; - -import static org.junit.Assert.assertTrue; - -public class QueryDataTest extends BaseTest { - - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "meters"; - static String host = "localhost"; - static int numOfTables = 30; - final static int numOfRecordsPerTable = 1000; - static long ts = 1496732686000l; - final static String tablePrefix = "t"; - - @Before - public void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - - String createTableSql = "create table " + stbName + "(ts timestamp, name binary(6))"; - statement.executeUpdate(createTableSql); - } - - @Test - public void testQueryBinaryData() throws SQLException{ - - String insertSql = "insert into " + stbName + " values(now, 'taosda')"; - System.out.println(insertSql); - - statement.executeUpdate(insertSql); - - String querySql = "select * from " + stbName; - ResultSet rs = statement.executeQuery(querySql); - - while(rs.next()) { - String name = rs.getString(2) + "001"; - System.out.println("name = " + name); - assertEquals(name, "taosda001"); - } - rs.close(); - } - - - @After - public void close() throws Exception { - statement.close(); - connection.close(); - Thread.sleep(10); - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java index 8067c547df148587766d87e204f5abad2bfb659d..87348f90261163eff650ecb225a168921c162f58 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java @@ -1,54 +1,49 @@ package com.taosdata.jdbc; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import javax.sql.rowset.serial.SerialBlob; import javax.sql.rowset.serial.SerialClob; +import java.io.UnsupportedEncodingException; import java.sql.*; import java.util.HashMap; import java.util.Properties; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -public class ResultSetTest extends BaseTest { - static Connection connection = null; - static Statement statement = null; +public class ResultSetTest { + static Connection connection; + static Statement statement; static String dbName = "test"; static String tName = "t0"; static String host = "localhost"; - static ResultSet resSet = null; + static ResultSet resSet; @BeforeClass - public static void createDatabaseAndTable() throws SQLException { + public static void createDatabaseAndTable() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.execute("use " + dbName); + statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k1 int, k2 bigint, k3 float, k4 double, k5 binary(30), k6 smallint, k7 bool, k8 nchar(20))"); + } catch (ClassNotFoundException | SQLException e) { return; } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + - " (ts timestamp, k1 int, k2 bigint, k3 float, k4 double, k5 binary(30), k6 smallint, k7 bool, k8 nchar(20))"); - - statement.executeQuery("use " + dbName); } @Test public void testResultSet() { - String sql = null; + String sql; long ts = 1496732686000l; int v1 = 2147483600; long v2 = ts + 1000; @@ -58,51 +53,38 @@ public class ResultSetTest extends BaseTest { short v6 = 12; boolean v7 = false; String v8 = "TDengine is powerful"; - sql = "insert into " + dbName + "." + tName + " values (" + ts + "," + v1 + "," + v2 + "," + v3 + "," + v4 + ",\"" + v5 + "\"," + v6 + "," + v7 + ",\"" + v8 + "\")"; - try { statement.executeUpdate(sql); assertEquals(1, statement.getUpdateCount()); } catch (SQLException e) { assert false : "insert error " + e.getMessage(); } - try { - statement.executeQuery("select * from " + dbName + "." + tName + " where ts = " + ts); + statement.execute("select * from " + dbName + "." + tName + " where ts = " + ts); resSet = statement.getResultSet(); System.out.println(((TSDBResultSet) resSet).getRowData()); while (resSet.next()) { assertEquals(ts, resSet.getLong(1)); assertEquals(ts, resSet.getLong("ts")); - System.out.println(resSet.getTimestamp(1)); - assertEquals(v1, resSet.getInt(2)); assertEquals(v1, resSet.getInt("k1")); - assertEquals(v2, resSet.getLong(3)); assertEquals(v2, resSet.getLong("k2")); - assertEquals(v3, resSet.getFloat(4), 7); assertEquals(v3, resSet.getFloat("k3"), 7); - assertEquals(v4, resSet.getDouble(5), 13); assertEquals(v4, resSet.getDouble("k4"), 13); - assertEquals(v5, resSet.getString(6)); assertEquals(v5, resSet.getString("k5")); - assertEquals(v6, resSet.getShort(7)); assertEquals(v6, resSet.getShort("k6")); - assertEquals(v7, resSet.getBoolean(8)); assertEquals(v7, resSet.getBoolean("k7")); - assertEquals(v8, resSet.getString(9)); assertEquals(v8, resSet.getString("k8")); - resSet.getBytes(9); resSet.getObject(6); resSet.getObject("k8"); @@ -115,692 +97,145 @@ public class ResultSetTest extends BaseTest { } } - @Test - public void testUnsupport() throws SQLException { - statement.executeQuery("show databases"); + @Test(expected = SQLException.class) + public void testUnsupport() throws SQLException, UnsupportedEncodingException { + statement.execute("show databases"); resSet = statement.getResultSet(); - try { - resSet.unwrap(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isWrapperFor(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getAsciiStream(0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getUnicodeStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBinaryStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getAsciiStream(""); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getUnicodeStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBinaryStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getWarnings(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.clearWarnings(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getCursorName(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isBeforeFirst(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isAfterLast(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isFirst(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isLast(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.beforeFirst(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.afterLast(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.first(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.last(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.absolute(1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.relative(1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.previous(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.setFetchDirection(0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getFetchDirection(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.setFetchSize(0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getFetchSize(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getConcurrency(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.rowUpdated(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.rowInserted(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.rowDeleted(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNull(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBoolean(0, true); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateByte(0, (byte) 2); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateShort(0, (short) 1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateInt(0, 0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateLong(0, 0l); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateFloat(0, 3.14f); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDouble(0, 3.1415); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBigDecimal(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBytes(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNull(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBoolean("", false); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateByte("", (byte) 1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateShort("", (short) 1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateInt("", 0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateLong("", 0l); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateFloat("", 3.14f); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDouble("", 3.1415); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBigDecimal(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBytes(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.insertRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.deleteRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.refreshRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.cancelRowUpdates(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.moveToInsertRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.moveToCurrentRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getStatement(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getObject(0, new HashMap<>()); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRef(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBlob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getArray(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getObject("", new HashMap<>()); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRef(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBlob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getArray(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getURL(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getURL(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRef(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRef(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBlob(0, new SerialBlob("".getBytes("UTF8"))); - } catch (Exception e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBlob("", new SerialBlob("".getBytes("UTF8"))); - } catch (Exception e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateClob("", new SerialClob("".toCharArray())); - } catch (Exception e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateClob(0, new SerialClob("".toCharArray())); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateArray(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateArray(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRowId(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRowId(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRowId(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRowId(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getHoldability(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - - try { - resSet.getNClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getNClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getSQLXML(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getSQLXML(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateSQLXML(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateSQLXML(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getNCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getNCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } + Assert.assertNotNull(resSet.unwrap(TSDBResultSet.class)); + Assert.assertTrue(resSet.isWrapperFor(TSDBResultSet.class)); + resSet.getUnicodeStream(null); + resSet.getBinaryStream(null); + resSet.getAsciiStream(""); + resSet.getUnicodeStream(null); + resSet.getBinaryStream(null); + resSet.getWarnings(); + resSet.clearWarnings(); + resSet.getCursorName(); + resSet.getCharacterStream(null); + resSet.getCharacterStream(null); + resSet.isBeforeFirst(); + resSet.isAfterLast(); + resSet.isFirst(); + resSet.isLast(); + resSet.beforeFirst(); + resSet.afterLast(); + resSet.first(); + resSet.last(); + resSet.getRow(); + resSet.absolute(1); + resSet.relative(1); + resSet.previous(); + resSet.setFetchDirection(0); + resSet.getFetchDirection(); + resSet.setFetchSize(0); + resSet.getFetchSize(); + resSet.getConcurrency(); + resSet.rowUpdated(); + resSet.rowInserted(); + resSet.rowDeleted(); + resSet.updateNull(null); + resSet.updateBoolean(0, true); + resSet.updateByte(0, (byte) 2); + resSet.updateShort(0, (short) 1); + resSet.updateInt(0, 0); + resSet.updateLong(0, 0l); + resSet.updateFloat(0, 3.14f); + resSet.updateDouble(0, 3.1415); + resSet.updateBigDecimal(null, null); + resSet.updateString(null, null); + resSet.updateBytes(null, null); + resSet.updateDate(null, null); + resSet.updateTime(null, null); + resSet.updateTimestamp(null, null); + resSet.updateAsciiStream(null, null); + resSet.updateBinaryStream(null, null); + resSet.updateCharacterStream(null, null); + resSet.updateObject(null, null); + resSet.updateObject(null, null); + resSet.updateNull(null); + resSet.updateBoolean("", false); + resSet.updateByte("", (byte) 1); + resSet.updateShort("", (short) 1); + resSet.updateInt("", 0); + resSet.updateLong("", 0l); + resSet.updateFloat("", 3.14f); + resSet.updateDouble("", 3.1415); + resSet.updateBigDecimal(null, null); + resSet.updateString(null, null); + resSet.updateBytes(null, null); + resSet.updateDate(null, null); + resSet.updateTime(null, null); + resSet.updateTimestamp(null, null); + resSet.updateAsciiStream(null, null); + resSet.updateBinaryStream(null, null); + resSet.updateCharacterStream(null, null); + resSet.updateObject(null, null); + resSet.updateObject(null, null); + resSet.insertRow(); + resSet.updateRow(); + resSet.deleteRow(); + resSet.refreshRow(); + resSet.cancelRowUpdates(); + resSet.moveToInsertRow(); + resSet.moveToCurrentRow(); + resSet.getStatement(); + resSet.getObject(0, new HashMap<>()); + resSet.getRef(null); + resSet.getBlob(null); + resSet.getClob(null); + resSet.getArray(null); + resSet.getObject("", new HashMap<>()); + resSet.getRef(null); + resSet.getBlob(null); + resSet.getClob(null); + resSet.getArray(null); + resSet.getDate(null, null); + resSet.getDate(null, null); + resSet.getTime(null, null); + resSet.getTime(null, null); + resSet.getTimestamp(null, null); + resSet.getTimestamp(null, null); + resSet.getURL(null); + resSet.getURL(null); + resSet.updateRef(null, null); + resSet.updateRef(null, null); + resSet.updateBlob(0, new SerialBlob("".getBytes("UTF8"))); + resSet.updateBlob("", new SerialBlob("".getBytes("UTF8"))); + resSet.updateClob("", new SerialClob("".toCharArray())); + resSet.updateClob(0, new SerialClob("".toCharArray())); + resSet.updateArray(null, null); + resSet.updateArray(null, null); + resSet.getRowId(null); + resSet.getRowId(null); + resSet.updateRowId(null, null); + resSet.updateRowId(null, null); + resSet.getHoldability(); + resSet.updateNString(null, null); + resSet.updateNString(null, null); + resSet.getNClob(null); + resSet.getNClob(null); + resSet.getSQLXML(null); + resSet.getSQLXML(null); + resSet.updateSQLXML(null, null); + resSet.updateSQLXML(null, null); + resSet.getNCharacterStream(null); + resSet.getNCharacterStream(null); + resSet.updateNCharacterStream(null, null); + resSet.updateNCharacterStream(null, null); + resSet.updateAsciiStream(null, null); + resSet.updateBinaryStream(null, null); + resSet.updateCharacterStream(null, null); + resSet.updateAsciiStream(null, null); + resSet.updateBinaryStream(null, null); + resSet.updateCharacterStream(null, null); + resSet.updateNCharacterStream(null, null); + resSet.updateNCharacterStream(null, null); + resSet.updateAsciiStream(null, null); + resSet.updateBinaryStream(null, null); + resSet.updateCharacterStream(null, null); + resSet.updateAsciiStream(null, null); + resSet.updateBinaryStream(null, null); + resSet.updateCharacterStream(null, null); } @Test @@ -815,13 +250,17 @@ public class ResultSetTest extends BaseTest { assertEquals(res.length, 2); statement.clearBatch(); } - @AfterClass - public static void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); + @AfterClass + public static void close() { + try { + statement.executeUpdate("drop database " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } } - } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java deleted file mode 100644 index 6c75860e0f0a2647175de6e836f7f1aeb62c539a..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; - -public class SelectTest extends BaseTest { - Connection connection = null; - Statement statement = null; - String dbName = "test"; - String tName = "t0"; - String host = "localhost"; - - @Before - public void createDatabaseAndTable() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - } - - @Test - public void selectData() throws SQLException { - long ts = 1496732686000l; - - for (int i = 0; i < 50; i++) { - ts++; - int row = statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); - System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); - assertEquals(1, row); - } - - String sql = "select * from " + dbName + "." + tName; - ResultSet resSet = statement.executeQuery(sql); - - int num = 0; - while (resSet.next()) { - num++; - } - resSet.close(); - - assertEquals(num, 50); - } - - @After - public void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java deleted file mode 100644 index 6e01fb7c349bbab73fcc6878d85330ac39de2cfc..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class StableTest extends BaseTest { - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "st"; - static String host = "localhost"; - - @BeforeClass - public static void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeQuery("use " + dbName); - } - -// @Test - public void createStable() { - String sql = "create table " + stbName + " (ts timestamp, v1 int, v2 int) tags (tg nchar(20)) "; - - try { - statement.executeUpdate(sql); - } catch (SQLException e) { - assert false : "error create stable" + e.getMessage(); - } - } - -// @Test - public void createTable() { - String sql = "create table t1 using " + stbName + " tags (\"beijing\")"; - - try { - statement.executeUpdate(sql); - } catch (SQLException e) { - assert false : "error create table" + e.getMessage(); - } - } - - @Test - public void describeSTable() { - createStable(); - String sql = "describe " + stbName; - int num = 0; - System.out.println("describe stable"); - try { - ResultSet res = statement.executeQuery(sql); - while (res.next()) { - for (int i = 1; i <= res.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s\n", i, res.getString(i)); - } - num++; - } - res.close(); - assertEquals(4, num); - } catch (SQLException e) { - assert false : "error describe stable" + e.getMessage(); - } - } - - @Test - public void describeTable() { - createTable(); - String sql = "describe t1"; - int num = 0; - System.out.println("describe table"); - try { - ResultSet res = statement.executeQuery(sql); - while (res.next()) { - for (int i = 1; i <= res.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s\n", i, res.getString(i)); - } - num++; - } - res.close(); - assertEquals(4, num); - } catch (SQLException e) { - assert false : "error describe stable" + e.getMessage(); - } - } - - // @Test - public void validCreateSql() { - String sql = "create table t2 using " + stbName + " tags (\"beijing\")"; - boolean valid = ((TSDBConnection) connection).getConnection().validateCreateTableSql(sql); - assertEquals(true, valid); - } - - @AfterClass - public static void close() throws Exception { - if (!statement.isClosed()) { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index db7b8c8cb14cbc0d1ae5a5a3a3bf8df590c4bc2a..73ceafa7299b256d7e83064b53bd638835a4b075 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -1,40 +1,51 @@ package com.taosdata.jdbc; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import java.sql.*; import java.util.Properties; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class StatementTest extends BaseTest { +public class StatementTest { static Connection connection = null; static Statement statement = null; static String dbName = "test"; static String tName = "t0"; static String host = "localhost"; - static ResultSet resSet = null; @BeforeClass public static void createConnection() throws SQLException { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root&password=taosdata", properties); + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + } catch (ClassNotFoundException e) { return; } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); + } + @Test + public void testCase() { + try { + ResultSet rs = statement.executeQuery("show databases"); + ResultSetMetaData metaData = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + System.out.print(metaData.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } } @Test @@ -44,130 +55,54 @@ public class StatementTest extends BaseTest { statement.executeUpdate("create database if not exists " + dbName); statement.executeUpdate("create table if not exists " + dbName + "." + tName + "(ts timestamp, k1 int)"); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); - statement.executeQuery("select * from " + dbName + "." + tName); + statement.execute("select * from " + dbName + "." + tName); ResultSet resultSet = statement.getResultSet(); - assertTrue(null != resultSet); + Assert.assertNotNull(resultSet); boolean isClosed = statement.isClosed(); - assertEquals(false, isClosed); + Assert.assertEquals(false, isClosed); } - @Test - public void testUnsupport() { -// if(null == resSet) { -// return; -// } - TSDBStatement tsdbStatement = (TSDBStatement) statement; - try { - tsdbStatement.unwrap(null); - } catch (SQLException e) { - } - try { - tsdbStatement.isWrapperFor(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMaxFieldSize(); - } catch (SQLException e) { - } - try { - tsdbStatement.setMaxFieldSize(0); - } catch (SQLException e) { - } - try { - tsdbStatement.setEscapeProcessing(true); - } catch (SQLException e) { - } - try { - tsdbStatement.cancel(); - } catch (SQLException e) { - } - try { - tsdbStatement.getWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.clearWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.setCursorName(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.setFetchDirection(0); - } catch (SQLException e) { - } - try { - tsdbStatement.getFetchDirection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetConcurrency(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetType(); - } catch (SQLException e) { - } - try { - tsdbStatement.getConnection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.getGeneratedKeys(); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, 0); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new int[]{0}); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"}); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetHoldability(); - } catch (SQLException e) { - } - try { - tsdbStatement.setPoolable(true); - } catch (SQLException e) { - } - try { - tsdbStatement.isPoolable(); - } catch (SQLException e) { - } - try { - tsdbStatement.closeOnCompletion(); - } catch (SQLException e) { - } - try { - tsdbStatement.isCloseOnCompletion(); - } catch (SQLException e) { - } + @Test(expected = SQLException.class) + public void testUnsupport() throws SQLException { + Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); + Assert.assertTrue(statement.isWrapperFor(TSDBStatement.class)); + + statement.getMaxFieldSize(); + statement.setMaxFieldSize(0); + statement.setEscapeProcessing(true); + statement.cancel(); + statement.getWarnings(); + statement.clearWarnings(); + statement.setCursorName(null); + statement.getMoreResults(); + statement.setFetchDirection(0); + statement.getFetchDirection(); + statement.getResultSetConcurrency(); + statement.getResultSetType(); + statement.getConnection(); + statement.getMoreResults(); + statement.getGeneratedKeys(); + statement.executeUpdate(null, 0); + statement.executeUpdate(null, new int[]{0}); + statement.executeUpdate(null, new String[]{"str1", "str2"}); + statement.getResultSetHoldability(); + statement.setPoolable(true); + statement.isPoolable(); + statement.closeOnCompletion(); + statement.isCloseOnCompletion(); } @AfterClass - public static void close() throws Exception { - if (!statement.isClosed()) { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - + public static void close() { + try { + statement.execute("drop database if exists " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); } } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 0a71c77d1d9dc718cc029c6a0b11ed63cd3034c1..11c3de305280a48b690ccdacd2df9751f4b718a9 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -9,80 +9,83 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; +import java.util.concurrent.TimeUnit; -public class SubscribeTest extends BaseTest { - Connection connection = null; - Statement statement = null; +public class SubscribeTest { + Connection connection; + Statement statement; String dbName = "test"; String tName = "t0"; - String host = "localhost"; + String host = "127.0.0.1"; String topic = "test"; @Before - public void createDatabase() throws SQLException { + public void createDatabase() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + statement = connection.createStatement(); + statement.execute("drop database if exists " + dbName); + statement.execute("create database if not exists " + dbName); + statement.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); + long ts = System.currentTimeMillis(); + for (int i = 0; i < 2; i++) { + ts += i; + String sql = "insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"; + statement.executeUpdate(sql); + } - statement = connection.createStatement(); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - long ts = System.currentTimeMillis(); - for (int i = 0; i < 2; i++) { - ts += i; - String sql = "insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"; - statement.executeUpdate(sql); + } catch (ClassNotFoundException | SQLException e) { + return; } } @Test - public void subscribe() throws Exception { - TSDBSubscribe subscribe = null; + public void subscribe() { try { - String rawSql = "select * from " + dbName + "." + tName + ";"; System.out.println(rawSql); - subscribe = ((TSDBConnection) connection).subscribe(topic, rawSql, false); - - int a = 0; - while (true) { - Thread.sleep(900); - TSDBResultSet resSet = subscribe.consume(); +// TSDBSubscribe subscribe = ((TSDBConnection) connection).subscribe(topic, rawSql, false); - while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); - } - System.out.println("\n======" + a + "=========="); - } - resSet.close(); - a++; - if (a >= 2) { - break; - } - } +// int a = 0; +// while (true) { +// TimeUnit.MILLISECONDS.sleep(1000); +// TSDBResultSet resSet = subscribe.consume(); +// while (resSet.next()) { +// for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { +// System.out.printf(i + ": " + resSet.getString(i) + "\t"); +// } +// System.out.println("\n======" + a + "=========="); +// } +// a++; +// if (a >= 2) { +// break; +// } +// resSet.close(); +// } +// +// subscribe.close(true); } catch (Exception e) { e.printStackTrace(); - } finally { - if (null != subscribe) { - subscribe.close(true); - } } } @After - public void close() throws Exception { - statement.executeQuery("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); + public void close() { + try { + statement.execute("drop database " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java new file mode 100644 index 0000000000000000000000000000000000000000..0a4ecb739cfd5a0abda01f7788d375ef95e0208a --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java @@ -0,0 +1,423 @@ +package com.taosdata.jdbc; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.management.OperationsException; +import java.sql.*; +import java.util.Properties; + +public class TSDBConnectionTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + + @Test + public void getConnection() { + // already test in beforeClass method + } + + @Test + public void createStatement() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void subscribe() { + try { + TSDBConnection unwrap = conn.unwrap(TSDBConnection.class); + TSDBSubscribe subscribe = unwrap.subscribe("topic1", "select * from log.log", false); + TSDBResultSet rs = subscribe.consume(); + ResultSetMetaData metaData = rs.getMetaData(); + for (int count = 0; count < 10 && rs.next(); count++) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String value = rs.getString(i); + System.out.print(metaData.getColumnLabel(i) + ":" + value + "\t"); + } + System.out.println(); + } + Assert.assertNotNull(rs); + subscribe.close(false); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Test + public void prepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()"); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void prepareCall() throws SQLException { + conn.prepareCall("select server_status()"); + } + + @Test + public void nativeSQL() throws SQLException { + String nativeSQL = conn.nativeSQL("select * from log.log"); + Assert.assertEquals("select * from log.log", nativeSQL); + } + + @Test + public void setAutoCommit() throws SQLException { + conn.setAutoCommit(true); + conn.setAutoCommit(false); + } + + @Test + public void getAutoCommit() throws SQLException { + Assert.assertTrue(conn.getAutoCommit()); + } + + @Test + public void commit() throws SQLException { + conn.commit(); + } + + @Test + public void rollback() throws SQLException { + conn.rollback(); + } + + @Test + public void close() { + // connection will close in afterClass method + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(conn.isClosed()); + } + + @Test + public void getMetaData() throws SQLException { + DatabaseMetaData meta = conn.getMetaData(); + Assert.assertNotNull(meta); + Assert.assertEquals("com.taosdata.jdbc.TSDBDriver", meta.getDriverName()); + } + + @Test + public void setReadOnly() throws SQLException { + conn.setReadOnly(true); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertTrue(conn.isReadOnly()); + } + + @Test + public void setCatalog() throws SQLException { + conn.setCatalog("test"); + Assert.assertEquals("test", conn.getCatalog()); + } + + @Test + public void getCatalog() throws SQLException { + conn.setCatalog("log"); + Assert.assertEquals("log", conn.getCatalog()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTransactionIsolation() throws SQLException { + conn.setTransactionIsolation(Connection.TRANSACTION_NONE); + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); + } + + @Test + public void getTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(conn.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + conn.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getTypeMap() throws SQLException { + conn.getTypeMap(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTypeMap() throws SQLException { + conn.setTypeMap(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setHoldability() throws SQLException { + conn.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + conn.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSavepoint() throws SQLException { + conn.setSavepoint(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testSetSavepoint() throws SQLException { + conn.setSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testRollback() throws SQLException { + conn.rollback(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void releaseSavepoint() throws SQLException { + conn.releaseSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement1() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement1() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall1() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement2() throws SQLException { + Assert.assertNotNull("", Statement.NO_GENERATED_KEYS); + conn.prepareStatement("", Statement.RETURN_GENERATED_KEYS); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement3() throws SQLException { + conn.prepareStatement("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement4() throws SQLException { + conn.prepareStatement("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createClob() throws SQLException { + conn.createClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createBlob() throws SQLException { + conn.createBlob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createNClob() throws SQLException { + conn.createNClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createSQLXML() throws SQLException { + conn.createSQLXML(); + } + + @Test(expected = SQLException.class) + public void isValid() throws SQLException { + Assert.assertTrue(conn.isValid(10)); + Assert.assertTrue(conn.isValid(0)); + conn.isValid(-1); + } + + @Test + public void setClientInfo() throws SQLClientInfoException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTC-8"); + } + + @Test + public void testSetClientInfo() throws SQLClientInfoException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn.setClientInfo(properties); + } + + @Test + public void getClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + Properties info = conn.getClientInfo(); + String charset = info.getProperty(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = info.getProperty(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = info.getProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test + public void testGetClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String charset = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createArrayOf() throws SQLException { + conn.createArrayOf("", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createStruct() throws SQLException { + conn.createStruct("", null); + } + + @Test + public void setSchema() throws SQLException { + conn.setSchema("test"); + } + + @Test + public void getSchema() throws SQLException { + Assert.assertNull(conn.getSchema()); + } + + @Test + public void abort() throws SQLException { + conn.abort(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNetworkTimeout() throws SQLException { + conn.setNetworkTimeout(null, 1000); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNetworkTimeout() throws SQLException { + conn.getNetworkTimeout(); + } + + @Test + public void unwrap() { + try { + TSDBConnection tsdbConnection = conn.unwrap(TSDBConnection.class); + Assert.assertNotNull(tsdbConnection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(conn.isWrapperFor(TSDBConnection.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/log?user=root&password=taosdata", properties); + // create test database for test cases + try (Statement stmt = conn.createStatement()) { + stmt.execute("create database if not exists test"); + } + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a7657fa3e581c96cf85555b16fca5f8b4c7f8ebb --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java @@ -0,0 +1,978 @@ +package com.taosdata.jdbc; + +import org.junit.*; + +import java.sql.*; +import java.util.Properties; + +public class TSDBDatabaseMetaDataTest { + private static final String host = "127.0.0.1"; + private static Connection connection; + private static TSDBDatabaseMetaData metaData; + + @Test + public void unwrap() throws SQLException { + TSDBDatabaseMetaData unwrap = metaData.unwrap(TSDBDatabaseMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(metaData.isWrapperFor(TSDBDatabaseMetaData.class)); + } + + @Test + public void allProceduresAreCallable() throws SQLException { + Assert.assertFalse(metaData.allProceduresAreCallable()); + } + + @Test + public void allTablesAreSelectable() throws SQLException { + Assert.assertFalse(metaData.allTablesAreSelectable()); + } + + @Test + public void getURL() throws SQLException { + Assert.assertEquals("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", metaData.getURL()); + } + + @Test + public void getUserName() throws SQLException { + Assert.assertEquals("root", metaData.getUserName()); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertFalse(metaData.isReadOnly()); + } + + @Test + public void nullsAreSortedHigh() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedHigh()); + } + + @Test + public void nullsAreSortedLow() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedLow()); + } + + @Test + public void nullsAreSortedAtStart() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedAtStart()); + } + + @Test + public void nullsAreSortedAtEnd() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedAtEnd()); + } + + @Test + public void getDatabaseProductName() throws SQLException { + Assert.assertEquals("TDengine", metaData.getDatabaseProductName()); + } + + @Test + public void getDatabaseProductVersion() throws SQLException { + Assert.assertEquals("2.0.x.x", metaData.getDatabaseProductVersion()); + } + + @Test + public void getDriverName() throws SQLException { + Assert.assertEquals("com.taosdata.jdbc.TSDBDriver", metaData.getDriverName()); + } + + @Test + public void getDriverVersion() throws SQLException { + Assert.assertEquals("2.0.x", metaData.getDriverVersion()); + } + + @Test + public void getDriverMajorVersion() { + Assert.assertEquals(2, metaData.getDriverMajorVersion()); + } + + @Test + public void getDriverMinorVersion() { + Assert.assertEquals(0, metaData.getDriverMinorVersion()); + } + + @Test + public void usesLocalFiles() throws SQLException { + Assert.assertFalse(metaData.usesLocalFiles()); + } + + @Test + public void usesLocalFilePerTable() throws SQLException { + Assert.assertFalse(metaData.usesLocalFilePerTable()); + } + + @Test + public void supportsMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseIdentifiers()); + } + + @Test + public void storesUpperCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseIdentifiers()); + } + + @Test + public void storesLowerCaseIdentifiers() throws SQLException { + Assert.assertTrue(metaData.storesLowerCaseIdentifiers()); + } + + @Test + public void storesMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseIdentifiers()); + } + + @Test + public void supportsMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseQuotedIdentifiers()); + } + + @Test + public void storesUpperCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseQuotedIdentifiers()); + } + + @Test + public void storesLowerCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesLowerCaseQuotedIdentifiers()); + } + + @Test + public void storesMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseQuotedIdentifiers()); + } + + @Test + public void getIdentifierQuoteString() throws SQLException { + Assert.assertEquals(" ", metaData.getIdentifierQuoteString()); + } + + @Test + public void getSQLKeywords() throws SQLException { + Assert.assertEquals(null, metaData.getSQLKeywords()); + } + + @Test + public void getNumericFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getNumericFunctions()); + } + + @Test + public void getStringFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getStringFunctions()); + } + + @Test + public void getSystemFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getSystemFunctions()); + } + + @Test + public void getTimeDateFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getTimeDateFunctions()); + } + + @Test + public void getSearchStringEscape() throws SQLException { + Assert.assertEquals(null, metaData.getSearchStringEscape()); + } + + @Test + public void getExtraNameCharacters() throws SQLException { + Assert.assertEquals(null, metaData.getExtraNameCharacters()); + } + + @Test + public void supportsAlterTableWithAddColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithAddColumn()); + } + + @Test + public void supportsAlterTableWithDropColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithDropColumn()); + } + + @Test + public void supportsColumnAliasing() throws SQLException { + Assert.assertTrue(metaData.supportsColumnAliasing()); + } + + @Test + public void nullPlusNonNullIsNull() throws SQLException { + Assert.assertFalse(metaData.nullPlusNonNullIsNull()); + } + + @Test + public void supportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert()); + } + + @Test + public void testSupportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert(1, 1)); + } + + @Test + public void supportsTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsTableCorrelationNames()); + } + + @Test + public void supportsDifferentTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsDifferentTableCorrelationNames()); + } + + @Test + public void supportsExpressionsInOrderBy() throws SQLException { + Assert.assertFalse(metaData.supportsExpressionsInOrderBy()); + } + + @Test + public void supportsOrderByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsOrderByUnrelated()); + } + + @Test + public void supportsGroupBy() throws SQLException { + Assert.assertTrue(metaData.supportsGroupBy()); + } + + @Test + public void supportsGroupByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByUnrelated()); + } + + @Test + public void supportsGroupByBeyondSelect() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByBeyondSelect()); + } + + @Test + public void supportsLikeEscapeClause() throws SQLException { + Assert.assertFalse(metaData.supportsLikeEscapeClause()); + } + + @Test + public void supportsMultipleResultSets() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleResultSets()); + } + + @Test + public void supportsMultipleTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleTransactions()); + } + + @Test + public void supportsNonNullableColumns() throws SQLException { + Assert.assertFalse(metaData.supportsNonNullableColumns()); + } + + @Test + public void supportsMinimumSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsMinimumSQLGrammar()); + } + + @Test + public void supportsCoreSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsCoreSQLGrammar()); + } + + @Test + public void supportsExtendedSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsExtendedSQLGrammar()); + } + + @Test + public void supportsANSI92EntryLevelSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92EntryLevelSQL()); + } + + @Test + public void supportsANSI92IntermediateSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92IntermediateSQL()); + } + + @Test + public void supportsANSI92FullSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92FullSQL()); + } + + @Test + public void supportsIntegrityEnhancementFacility() throws SQLException { + Assert.assertFalse(metaData.supportsIntegrityEnhancementFacility()); + } + + @Test + public void supportsOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsOuterJoins()); + } + + @Test + public void supportsFullOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsFullOuterJoins()); + } + + @Test + public void supportsLimitedOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsLimitedOuterJoins()); + } + + @Test + public void getSchemaTerm() throws SQLException { + Assert.assertNull(metaData.getSchemaTerm()); + } + + @Test + public void getProcedureTerm() throws SQLException { + Assert.assertNull(metaData.getProcedureTerm()); + } + + @Test + public void getCatalogTerm() throws SQLException { + Assert.assertEquals("database", metaData.getCatalogTerm()); + } + + @Test + public void isCatalogAtStart() throws SQLException { + Assert.assertTrue(metaData.isCatalogAtStart()); + } + + @Test + public void getCatalogSeparator() throws SQLException { + Assert.assertEquals(".", metaData.getCatalogSeparator()); + } + + @Test + public void supportsSchemasInDataManipulation() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInDataManipulation()); + } + + @Test + public void supportsSchemasInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInProcedureCalls()); + } + + @Test + public void supportsSchemasInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInTableDefinitions()); + } + + @Test + public void supportsSchemasInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInIndexDefinitions()); + } + + @Test + public void supportsSchemasInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInPrivilegeDefinitions()); + } + + @Test + public void supportsCatalogsInDataManipulation() throws SQLException { + Assert.assertTrue(metaData.supportsCatalogsInDataManipulation()); + } + + @Test + public void supportsCatalogsInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInProcedureCalls()); + } + + @Test + public void supportsCatalogsInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInTableDefinitions()); + } + + @Test + public void supportsCatalogsInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInIndexDefinitions()); + } + + @Test + public void supportsCatalogsInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInPrivilegeDefinitions()); + } + + @Test + public void supportsPositionedDelete() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedDelete()); + } + + @Test + public void supportsPositionedUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedUpdate()); + } + + @Test + public void supportsSelectForUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsSelectForUpdate()); + } + + @Test + public void supportsStoredProcedures() throws SQLException { + Assert.assertFalse(metaData.supportsStoredProcedures()); + } + + @Test + public void supportsSubqueriesInComparisons() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInComparisons()); + } + + @Test + public void supportsSubqueriesInExists() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInExists()); + } + + @Test + public void supportsSubqueriesInIns() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInIns()); + } + + @Test + public void supportsSubqueriesInQuantifieds() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInQuantifieds()); + } + + @Test + public void supportsCorrelatedSubqueries() throws SQLException { + Assert.assertFalse(metaData.supportsCorrelatedSubqueries()); + } + + @Test + public void supportsUnion() throws SQLException { + Assert.assertFalse(metaData.supportsUnion()); + } + + @Test + public void supportsUnionAll() throws SQLException { + Assert.assertFalse(metaData.supportsUnionAll()); + } + + @Test + public void supportsOpenCursorsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossCommit()); + } + + @Test + public void supportsOpenCursorsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossRollback()); + } + + @Test + public void supportsOpenStatementsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossCommit()); + } + + @Test + public void supportsOpenStatementsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossRollback()); + } + + @Test + public void getMaxBinaryLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxBinaryLiteralLength()); + } + + @Test + public void getMaxCharLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCharLiteralLength()); + } + + @Test + public void getMaxColumnNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnNameLength()); + } + + @Test + public void getMaxColumnsInGroupBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInGroupBy()); + } + + @Test + public void getMaxColumnsInIndex() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInIndex()); + } + + @Test + public void getMaxColumnsInOrderBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInOrderBy()); + } + + @Test + public void getMaxColumnsInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInSelect()); + } + + @Test + public void getMaxColumnsInTable() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInTable()); + } + + @Test + public void getMaxConnections() throws SQLException { + Assert.assertEquals(0, metaData.getMaxConnections()); + } + + @Test + public void getMaxCursorNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCursorNameLength()); + } + + @Test + public void getMaxIndexLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxIndexLength()); + } + + @Test + public void getMaxSchemaNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxSchemaNameLength()); + } + + @Test + public void getMaxProcedureNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxProcedureNameLength()); + } + + @Test + public void getMaxCatalogNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCatalogNameLength()); + } + + @Test + public void getMaxRowSize() throws SQLException { + Assert.assertEquals(0, metaData.getMaxRowSize()); + } + + @Test + public void doesMaxRowSizeIncludeBlobs() throws SQLException { + Assert.assertFalse(metaData.doesMaxRowSizeIncludeBlobs()); + } + + @Test + public void getMaxStatementLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatementLength()); + } + + @Test + public void getMaxStatements() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatements()); + } + + @Test + public void getMaxTableNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTableNameLength()); + } + + @Test + public void getMaxTablesInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTablesInSelect()); + } + + @Test + public void getMaxUserNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxUserNameLength()); + } + + @Test + public void getDefaultTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, metaData.getDefaultTransactionIsolation()); + } + + @Test + public void supportsTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsTransactions()); + } + + @Test + public void supportsTransactionIsolationLevel() throws SQLException { + Assert.assertTrue(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_NONE)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_COMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_UNCOMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_REPEATABLE_READ)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_SERIALIZABLE)); + } + + @Test + public void supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsDataDefinitionAndDataManipulationTransactions()); + } + + @Test + public void supportsDataManipulationTransactionsOnly() throws SQLException { + Assert.assertFalse(metaData.supportsDataManipulationTransactionsOnly()); + } + + @Test + public void dataDefinitionCausesTransactionCommit() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionCausesTransactionCommit()); + } + + @Test + public void dataDefinitionIgnoredInTransactions() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionIgnoredInTransactions()); + } + + @Test + public void getProcedures() throws SQLException { + Assert.assertNull(metaData.getProcedures("*", "*", "*")); + } + + @Test + public void getProcedureColumns() throws SQLException { + Assert.assertNull(metaData.getProcedureColumns("*", "*", "*", "*")); + } + + @Test + public void getTables() throws SQLException { + System.out.println("****************************************************"); + ResultSet tables = metaData.getTables("log", "", null, null); + ResultSetMetaData metaData = tables.getMetaData(); + while (tables.next()) { + System.out.print(metaData.getColumnLabel(1) + ":" + tables.getString(1) + "\t"); + System.out.print(metaData.getColumnLabel(3) + ":" + tables.getString(3) + "\t"); + System.out.print(metaData.getColumnLabel(4) + ":" + tables.getString(4) + "\t"); + System.out.print(metaData.getColumnLabel(5) + ":" + tables.getString(5) + "\n"); + } + System.out.println(); + Assert.assertNotNull(tables); + } + + @Test + public void getSchemas() throws SQLException { + Assert.assertNotNull(metaData.getSchemas()); + } + + @Test + public void getCatalogs() throws SQLException { + System.out.println("****************************************************"); + + ResultSet catalogs = metaData.getCatalogs(); + ResultSetMetaData meta = catalogs.getMetaData(); + while (catalogs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + catalogs.getString(i)); + } + System.out.println(); + } + } + + @Test + public void getTableTypes() throws SQLException { + System.out.println("****************************************************"); + + ResultSet tableTypes = metaData.getTableTypes(); + while (tableTypes.next()) { + System.out.println(tableTypes.getString("TABLE_TYPE")); + } + Assert.assertNotNull(metaData.getTableTypes()); + } + + @Test + public void getColumns() throws SQLException { + System.out.println("****************************************************"); + + ResultSet columns = metaData.getColumns("log", "", "dn", ""); + ResultSetMetaData meta = columns.getMetaData(); + while (columns.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + columns.getString(1) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + columns.getString(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + columns.getString(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + columns.getString(5) + "\t"); + System.out.print(meta.getColumnLabel(6) + ": " + columns.getString(6) + "\t"); + System.out.print(meta.getColumnLabel(7) + ": " + columns.getString(7) + "\t"); + System.out.print(meta.getColumnLabel(9) + ": " + columns.getString(9) + "\t"); + System.out.print(meta.getColumnLabel(10) + ": " + columns.getString(10) + "\t"); + System.out.print(meta.getColumnLabel(11) + ": " + columns.getString(11) + "\n"); + System.out.print(meta.getColumnLabel(12) + ": " + columns.getString(12) + "\n"); + } + } + + @Test + public void getColumnPrivileges() throws SQLException { + Assert.assertNotNull(metaData.getColumnPrivileges("", "", "", "")); + } + + @Test + public void getTablePrivileges() throws SQLException { + Assert.assertNotNull(metaData.getTablePrivileges("", "", "")); + } + + @Test + public void getBestRowIdentifier() throws SQLException { + Assert.assertNotNull(metaData.getBestRowIdentifier("", "", "", 0, false)); + } + + @Test + public void getVersionColumns() throws SQLException { + Assert.assertNotNull(metaData.getVersionColumns("", "", "")); + } + + @Test + public void getPrimaryKeys() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getPrimaryKeys("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("COLUMN_NAME: " + rs.getString("COLUMN_NAME")); + System.out.println("KEY_SEQ: " + rs.getString("KEY_SEQ")); + System.out.println("PK_NAME: " + rs.getString("PK_NAME")); + } + + Assert.assertNotNull(rs); + } + + @Test + public void getImportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getImportedKeys("", "", "")); + } + + @Test + public void getExportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getExportedKeys("", "", "")); + } + + @Test + public void getCrossReference() throws SQLException { + Assert.assertNotNull(metaData.getCrossReference("", "", "", "", "", "")); + } + + @Test + public void getTypeInfo() throws SQLException { + Assert.assertNotNull(metaData.getTypeInfo()); + } + + @Test + public void getIndexInfo() throws SQLException { + Assert.assertNotNull(metaData.getIndexInfo("", "", "", false, false)); + } + + @Test + public void supportsResultSetType() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetType(0)); + } + + @Test + public void supportsResultSetConcurrency() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetConcurrency(0, 0)); + } + + @Test + public void ownUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownUpdatesAreVisible(0)); + } + + @Test + public void ownDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownDeletesAreVisible(0)); + } + + @Test + public void ownInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownInsertsAreVisible(0)); + } + + @Test + public void othersUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersUpdatesAreVisible(0)); + } + + @Test + public void othersDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersDeletesAreVisible(0)); + } + + @Test + public void othersInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersInsertsAreVisible(0)); + } + + @Test + public void updatesAreDetected() throws SQLException { + Assert.assertFalse(metaData.updatesAreDetected(0)); + } + + @Test + public void deletesAreDetected() throws SQLException { + Assert.assertFalse(metaData.deletesAreDetected(0)); + } + + @Test + public void insertsAreDetected() throws SQLException { + Assert.assertFalse(metaData.insertsAreDetected(0)); + } + + @Test + public void supportsBatchUpdates() throws SQLException { + Assert.assertFalse(metaData.supportsBatchUpdates()); + } + + @Test + public void getUDTs() throws SQLException { + Assert.assertNotNull(metaData.getUDTs("", "", "", null)); + } + + @Test + public void getConnection() throws SQLException { + Assert.assertNotNull(metaData.getConnection()); + } + + @Test + public void supportsSavepoints() throws SQLException { + Assert.assertFalse(metaData.supportsSavepoints()); + } + + @Test + public void supportsNamedParameters() throws SQLException { + Assert.assertFalse(metaData.supportsNamedParameters()); + } + + @Test + public void supportsMultipleOpenResults() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleOpenResults()); + } + + @Test + public void supportsGetGeneratedKeys() throws SQLException { + Assert.assertFalse(metaData.supportsGetGeneratedKeys()); + } + + @Test + public void getSuperTypes() throws SQLException { + Assert.assertNotNull(metaData.getSuperTypes("", "", "")); + } + + @Test + public void getSuperTables() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getSuperTables("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("SUPERTABLE_NAME: " + rs.getString("SUPERTABLE_NAME")); + } + Assert.assertNotNull(rs); + } + + @Test + public void getAttributes() throws SQLException { + Assert.assertNotNull(metaData.getAttributes("", "", "", "")); + } + + @Test + public void supportsResultSetHoldability() throws SQLException { + Assert.assertTrue(metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT)); + Assert.assertFalse(metaData.supportsResultSetHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT)); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(1, metaData.getResultSetHoldability()); + } + + @Test + public void getDatabaseMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getDatabaseMajorVersion()); + } + + @Test + public void getDatabaseMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getDatabaseMinorVersion()); + } + + @Test + public void getJDBCMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getJDBCMajorVersion()); + } + + @Test + public void getJDBCMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getJDBCMinorVersion()); + } + + @Test + public void getSQLStateType() throws SQLException { + Assert.assertEquals(0, metaData.getSQLStateType()); + } + + @Test + public void locatorsUpdateCopy() throws SQLException { + Assert.assertFalse(metaData.locatorsUpdateCopy()); + } + + @Test + public void supportsStatementPooling() throws SQLException { + Assert.assertFalse(metaData.supportsStatementPooling()); + } + + @Test + public void getRowIdLifetime() throws SQLException { + Assert.assertNull(metaData.getRowIdLifetime()); + } + + @Test + public void supportsStoredFunctionsUsingCallSyntax() throws SQLException { + Assert.assertFalse(metaData.supportsStoredFunctionsUsingCallSyntax()); + } + + @Test + public void autoCommitFailureClosesAllResultSets() throws SQLException { + Assert.assertFalse(metaData.autoCommitFailureClosesAllResultSets()); + } + + @Test + public void getClientInfoProperties() throws SQLException { + Assert.assertNotNull(metaData.getClientInfoProperties()); + } + + @Test + public void getFunctions() throws SQLException { + Assert.assertNotNull(metaData.getFunctions("", "", "")); + } + + @Test + public void getFunctionColumns() throws SQLException { + Assert.assertNotNull(metaData.getFunctionColumns("", "", "", "")); + } + + @Test + public void getPseudoColumns() throws SQLException { + Assert.assertNotNull(metaData.getPseudoColumns("", "", "", "")); + } + + @Test + public void generatedKeyAlwaysReturned() throws SQLException { + Assert.assertFalse(metaData.generatedKeyAlwaysReturned()); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", properties); + metaData = connection.getMetaData().unwrap(TSDBDatabaseMetaData.class); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 8adcdefb2974d5817793297091eee1c2f40a52b9..671ecd723d6fea8d6b9b8ccf94cba06689ce26b8 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -3,9 +3,6 @@ package com.taosdata.jdbc; import org.junit.BeforeClass; import org.junit.Test; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; import java.sql.*; import java.util.Properties; @@ -27,54 +24,14 @@ public class TSDBDriverTest { "jdbc:TAOS://:/test", "jdbc:TAOS://localhost:0/?user=root&password=taosdata" }; - private static boolean islibLoaded = false; - private static boolean isTaosdActived; - private Connection conn; - @BeforeClass - public static void before() { - String osName = System.getProperty("os.name").toLowerCase(); - if (!osName.equals("linux")) - return; - // try to load taos lib - try { - System.loadLibrary("taos"); - islibLoaded = true; - } catch (UnsatisfiedLinkError error) { - System.out.println("load tdengine lib failed."); - error.printStackTrace(); - } - // check taosd is activated - try { - String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""}; - Process exec = Runtime.getRuntime().exec(cmd); - BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); - int lineCnt = 0; - while (reader.readLine() != null) { - lineCnt++; - } - if (lineCnt > 0) - isTaosdActived = true; - } catch (IOException e) { - e.printStackTrace(); - } - - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - @Test - public void testConnectWithJdbcURL() { + public void connectWithJdbcURL() { final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(url); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(url); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -82,17 +39,15 @@ public class TSDBDriverTest { } @Test - public void testConnectWithProperties() { + public void connectWithProperties() { final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(jdbcUrl, connProps); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -100,33 +55,21 @@ public class TSDBDriverTest { } @Test - public void testConnectWithConfigFile() { + public void connectWithConfigFile() { String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(jdbcUrl, connProps); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); } } - @Test(expected = SQLException.class) - public void testAcceptsURL() throws SQLException { - Driver driver = new TSDBDriver(); - for (String url : validURLs) { - assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); - } - driver.acceptsURL(null); - fail("acceptsURL throws exception when parameter is null"); - } - @Test public void testParseURL() { TSDBDriver driver = new TSDBDriver(); @@ -167,8 +110,19 @@ public class TSDBDriverTest { assertNull("failure - dbname should be null", actual.getProperty("dbname")); } + + @Test(expected = SQLException.class) + public void acceptsURL() throws SQLException { + Driver driver = new TSDBDriver(); + for (String url : validURLs) { + assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); + } + driver.acceptsURL(null); + fail("acceptsURL throws exception when parameter is null"); + } + @Test - public void testGetPropertyInfo() throws SQLException { + public void getPropertyInfo() throws SQLException { Driver driver = new TSDBDriver(); final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; Properties connProps = new Properties(); @@ -188,23 +142,31 @@ public class TSDBDriverTest { } @Test - public void testGetMajorVersion() { - assertEquals("failure - getMajorVersion should be 2", 2, new TSDBDriver().getMajorVersion()); + public void getMajorVersion() { + assertEquals(2, new TSDBDriver().getMajorVersion()); } @Test - public void testGetMinorVersion() { - assertEquals("failure - getMinorVersion should be 0", 0, new TSDBDriver().getMinorVersion()); + public void getMinorVersion() { + assertEquals(0, new TSDBDriver().getMinorVersion()); } @Test - public void testJdbcCompliant() { - assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant()); + public void jdbcCompliant() { + assertFalse(new TSDBDriver().jdbcCompliant()); } @Test - public void testGetParentLogger() throws SQLFeatureNotSupportedException { - assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); + public void getParentLogger() throws SQLFeatureNotSupportedException { + assertNull(new TSDBDriver().getParentLogger()); } + @BeforeClass + public static void before() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..475cfef060e2769d3390c19bfebc521ada7c0b46 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -0,0 +1,242 @@ +package com.taosdata.jdbc; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class TSDBPreparedStatementTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and temperature >= ?"; + private static PreparedStatement pstmt_select; + + @Test + public void executeQuery() throws SQLException { + long end = System.currentTimeMillis(); + long start = end - 1000 * 60 * 60; + pstmt_select.setTimestamp(1, new Timestamp(start)); + pstmt_select.setTimestamp(2, new Timestamp(end)); + pstmt_select.setFloat(3, 0); + + ResultSet rs = pstmt_select.executeQuery(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + } + + @Test + public void executeUpdate() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(2, 3.14f); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + } + + @Test + public void setNull() throws SQLException { + pstmt_insert.setNull(2, Types.FLOAT); + } + + @Test + public void setBoolean() throws SQLException { + pstmt_insert.setBoolean(2, true); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setByte() throws SQLException { + pstmt_insert.setByte(1, (byte) 0x001); + } + + @Test + public void setShort() { + + } + + @Test + public void setInt() { + + } + + @Test + public void setLong() { + + } + + @Test + public void setFloat() { + + } + + @Test + public void setDouble() { + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBigDecimal() throws SQLException { + pstmt_insert.setBigDecimal(1, null); + } + + @Test + public void setString() { + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBytes() throws SQLException { + pstmt_insert.setBytes(1, new byte[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setDate() throws SQLException { + pstmt_insert.setDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTime() throws SQLException { + pstmt_insert.setTime(1, new Time(System.currentTimeMillis())); + } + + @Test + public void setTimestamp() { + //TODO + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setAsciiStream() throws SQLException { + pstmt_insert.setAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBinaryStream() throws SQLException { + pstmt_insert.setBinaryStream(1, null); + } + + @Test + public void clearParameters() { + //TODO + } + + @Test + public void setObject() throws SQLException { + pstmt_insert.setObject(1, System.currentTimeMillis()); + //TODO + } + + @Test + public void execute() { + //TODO + } + + @Test + public void addBatch() { + //TODO: + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCharacterStream() throws SQLException { + pstmt_insert.setCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRef() throws SQLException { + pstmt_insert.setRef(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBlob() throws SQLException { + pstmt_insert.setBlob(1, (Blob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setClob() throws SQLException { + pstmt_insert.setClob(1, (Clob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setArray() throws SQLException { + pstmt_insert.setArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getMetaData() throws SQLException { + pstmt_insert.getMetaData(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setURL() throws SQLException { + pstmt_insert.setURL(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getParameterMetaData() throws SQLException { + ParameterMetaData parameterMetaData = pstmt_insert.getParameterMetaData(); +// Assert.assertNotNull(parameterMetaData); + //TODO: + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRowId() throws SQLException { + pstmt_insert.setRowId(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNString() throws SQLException { + pstmt_insert.setNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNCharacterStream() throws SQLException { + pstmt_insert.setNCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNClob() throws SQLException { + pstmt_insert.setNClob(1, (NClob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSQLXML() throws SQLException { + pstmt_insert.setSQLXML(1, null); + } + + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + pstmt_select = conn.prepareStatement(sql_select); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..671551c42639620b09ace1ae315da6f25ea278d4 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java @@ -0,0 +1,415 @@ +package com.taosdata.jdbc; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; +import java.util.UUID; + +public class TSDBStatementTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + private static Statement stmt; + + @Test + public void executeQuery() { + try { + ResultSet rs = stmt.executeQuery("show databases"); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeUpdate() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + int affectRows = stmt.executeUpdate("create database " + dbName); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(1, affectRows); + affectRows = stmt.executeUpdate("drop database " + dbName); + Assert.assertEquals(0, affectRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void close() { + // test in AfterClass method + } + + + @Test + public void getMaxFieldSize() throws SQLException { + Assert.assertEquals(16 * 1024, stmt.getMaxFieldSize()); + } + + @Test(expected = SQLException.class) + public void setMaxFieldSize() throws SQLException { + + stmt.setMaxFieldSize(0); + stmt.setMaxFieldSize(-1); + } + + @Test + public void getMaxRows() throws SQLException { + Assert.assertEquals(0, stmt.getMaxRows()); + } + + @Test(expected = SQLException.class) + public void setMaxRows() throws SQLException { + stmt.setMaxRows(0); + stmt.setMaxRows(-1); + } + + @Test + public void setEscapeProcessing() throws SQLException { + stmt.setEscapeProcessing(true); + stmt.setEscapeProcessing(false); + } + + @Test + public void getQueryTimeout() throws SQLException { + Assert.assertEquals(0, stmt.getQueryTimeout()); + } + + @Test(expected = SQLException.class) + public void setQueryTimeout() throws SQLException { + stmt.setQueryTimeout(0); + stmt.setQueryTimeout(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancel() throws SQLException { + stmt.cancel(); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(stmt.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + stmt.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCursorName() throws SQLException { + stmt.setCursorName(""); + } + + @Test + public void execute() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getResultSet() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + ResultSet rs = stmt.getResultSet(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertEquals(3, meta.getColumnCount()); + int count = 0; + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + count++; + } + Assert.assertEquals(1, count); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getUpdateCount() { + // already test in execute method + } + + @Test + public void getMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults()); + } + + @Test(expected = SQLException.class) + public void setFetchDirection() throws SQLException { + stmt.setFetchDirection(ResultSet.FETCH_FORWARD); + stmt.setFetchDirection(ResultSet.FETCH_REVERSE); + stmt.setFetchDirection(ResultSet.FETCH_UNKNOWN); + stmt.setFetchDirection(-1); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, stmt.getFetchDirection()); + } + + @Test(expected = SQLException.class) + public void setFetchSize() throws SQLException { + stmt.setFetchSize(0); + stmt.setFetchSize(-1); + } + + @Test + public void getFetchSize() throws SQLException { + stmt.setFetchSize(0); + Assert.assertEquals(0, stmt.getFetchSize()); + stmt.setFetchSize(0); + } + + @Test + public void getResultSetConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, stmt.getResultSetConcurrency()); + } + + @Test + public void getResultSetType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, stmt.getResultSetType()); + } + + @Test + public void addBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void clearBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.clearBatch(); + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + stmt.clearBatch(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + int[] results = stmt.executeBatch(); + Assert.assertEquals(0, results[0]); + Assert.assertEquals(0, results[1]); + Assert.assertEquals(1, results[2]); + Assert.assertEquals(Statement.SUCCESS_NO_INFO, results[3]); + Assert.assertEquals(0, results[4]); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getConnection() { + try { + Connection connection = stmt.getConnection(); + Assert.assertNotNull(connection); + Assert.assertTrue(this.conn == connection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testGetMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults(Statement.CLOSE_CURRENT_RESULT)); + stmt.getMoreResults(Statement.KEEP_CURRENT_RESULT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getGeneratedKeys() throws SQLException { + stmt.getGeneratedKeys(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate() throws SQLException { + stmt.executeUpdate("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate1() throws SQLException { + stmt.executeUpdate("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate2() throws SQLException { + stmt.executeUpdate("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute() throws SQLException { + stmt.execute("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute1() throws SQLException { + stmt.execute("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute2() throws SQLException { + stmt.execute("", new String[]{}); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, stmt.getResultSetHoldability()); + } + + @Test + public void isClosed() { + try { + Assert.assertEquals(false, stmt.isClosed()); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void setPoolable() throws SQLException { + stmt.setPoolable(true); + stmt.setPoolable(false); + } + + @Test + public void isPoolable() throws SQLException { + Assert.assertEquals(false, stmt.isPoolable()); + } + + @Test + public void closeOnCompletion() throws SQLException { + stmt.closeOnCompletion(); + } + + @Test + public void isCloseOnCompletion() throws SQLException { + Assert.assertFalse(stmt.isCloseOnCompletion()); + } + + @Test + public void unwrap() throws SQLException { + TSDBStatement unwrap = stmt.unwrap(TSDBStatement.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(stmt.isWrapperFor(TSDBStatement.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", properties); + stmt = conn.createStatement(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java index 8de2e3b442b9b2ea1b84687878ba3bb4faf5e4d6..19bc5f713f9b406a943fc640fd03bb0503ed2967 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -19,7 +19,7 @@ public class AppMemoryLeakTest { } } - @Test + @Test(expected = Exception.class) public void testCreateTooManyStatement() throws ClassNotFoundException, SQLException { Class.forName("com.taosdata.jdbc.TSDBDriver"); int stmtCnt = 0; @@ -30,15 +30,4 @@ public class AppMemoryLeakTest { } } - public static void main(String[] args) throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - int stmtCnt = 0; - Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); - while (true) { - Statement stmt = conn.createStatement(); - System.out.println(++stmtCnt + " : " + stmt); - } - } - - } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java index 9608c4985db15312e8e2966badfe4db8241bc306..472da34980d98cbe694783b120bc5533a908fa5b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java @@ -1,14 +1,12 @@ package com.taosdata.jdbc.cases; -import com.taosdata.jdbc.lib.TSDBCommon; +import com.taosdata.jdbc.TSDBDriver; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; +import java.util.Properties; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -18,57 +16,71 @@ import static org.junit.Assert.assertEquals; public class BatchInsertTest { - static String host = "localhost"; + static String host = "127.0.0.1"; static String dbName = "test"; static String stbName = "meters"; static int numOfTables = 30; final static int numOfRecordsPerTable = 1000; static long ts = 1496732686000l; final static String tablePrefix = "t"; - private Connection connection; @Before public void before() { try { - connection = TSDBCommon.getConn(host); - TSDBCommon.createDatabase(connection, dbName); - TSDBCommon.createStable(connection, stbName); - TSDBCommon.createTables(connection, numOfTables, stbName, tablePrefix); + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + Statement statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.executeUpdate("use " + dbName); + // create stable + String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; + statement.executeUpdate(createTableSql); + // create tables + for(int i = 0; i < numOfTables; i++) { + String loc = i % 2 == 0 ? "beijing" : "shanghai"; + String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; + statement.executeUpdate(createSubTalbesSql); + } + statement.close(); } catch (Exception e) { e.printStackTrace(); } } @Test - public void testBatchInsert(){ + public void testBatchInsert() { ExecutorService executorService = Executors.newFixedThreadPool(numOfTables); for (int i = 0; i < numOfTables; i++) { final int index = i; - executorService.execute(new Runnable() { - @Override - public void run() { - try { - long startTime = System.currentTimeMillis(); - Statement statement = connection.createStatement(); // get statement - StringBuilder sb = new StringBuilder(); - sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); - Random rand = new Random(); - for (int j = 1; j <= numOfRecordsPerTable; j++) { - sb.append("(" + (ts + j) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ")"); - } - statement.addBatch(sb.toString()); - statement.executeBatch(); - long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); - connection.commit(); - statement.close(); - } catch (Exception e) { - e.printStackTrace(); + executorService.execute(() -> { + try { + long startTime = System.currentTimeMillis(); + Statement statement = connection.createStatement(); // get statement + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); + Random rand = new Random(); + for (int j = 1; j <= numOfRecordsPerTable; j++) { + sb.append("(" + (ts + j) + ", "); + sb.append(rand.nextInt(100) + ", "); + sb.append(rand.nextInt(100) + ", "); + sb.append(rand.nextInt(100) + ")"); } + statement.addBatch(sb.toString()); + statement.executeBatch(); + long endTime = System.currentTimeMillis(); + System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); + connection.commit(); + statement.close(); + } catch (Exception e) { + e.printStackTrace(); } }); } @@ -80,7 +92,7 @@ public class BatchInsertTest { e.printStackTrace(); } - try{ + try { Statement statement = connection.createStatement(); ResultSet rs = statement.executeQuery("select * from meters"); int num = 0; @@ -89,7 +101,7 @@ public class BatchInsertTest { } assertEquals(num, numOfTables * numOfRecordsPerTable); rs.close(); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } @@ -102,7 +114,6 @@ public class BatchInsertTest { } catch (SQLException e) { e.printStackTrace(); } - } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d8d8f1eae42f555b2bc0be12e8a3a943071da3c6 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java @@ -0,0 +1,24 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBErrorNumbers; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.DriverManager; +import java.sql.SQLException; + +public class ConnectWrongDatabaseTest { + + @Test + public void connect() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + DriverManager.getConnection("jdbc:TAOS://localhost:6030/wrong_db?user=root&password=taosdata"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + Assert.assertEquals(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL, e.getErrorCode()); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d7603312a090bedb17ed125edf6da535924964d0 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java @@ -0,0 +1,113 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.*; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class ImportTest { + private static Connection connection; + static String dbName = "test_import"; + static String tName = "t0"; + static String host = "127.0.0.1"; + private static long ts; + + @BeforeClass + public static void before() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + Statement stmt = connection.createStatement(); + stmt.execute("create database if not exists " + dbName); + stmt.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); + stmt.close(); + + ts = System.currentTimeMillis(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Test + public void case001_insertData() throws Exception { + try (Statement stmt = connection.createStatement()) { + for (int i = 0; i < 50; i++) { + ts++; + int row = stmt.executeUpdate("import into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); + System.out.println("import into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); + assertEquals(1, row); + } + } + } + + @Test + public void case002_checkSum() { + Assert.assertEquals(50, select()); + } + + private int select() { + int count = 0; + try (Statement stmt = connection.createStatement()) { + + String sql = "select * from " + dbName + "." + tName; + ResultSet rs = stmt.executeQuery(sql); + while (rs.next()) { + count++; + } + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + return count; + } + + @Test + public void case003_importData() { + // 避免时间重复 + try (Statement stmt = connection.createStatement()) { + StringBuilder sqlBuilder = new StringBuilder("import into ").append(dbName).append(".").append(tName).append(" values "); + for (int i = 0; i < 50; i++) { + int a = i / 5; + long t = (++ts) + a; + sqlBuilder.append("(").append(t).append(",").append((100 + i)).append(",").append(i).append(") "); + } + System.out.println(sqlBuilder.toString()); + int rows = stmt.executeUpdate(sqlBuilder.toString()); + assertEquals(50, rows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void case004_checkSum() { + Assert.assertEquals(100, select()); + } + + @AfterClass + public static void close() { + try { + if (connection != null) { + Statement statement = connection.createStatement(); + statement.executeUpdate("drop database " + dbName); + statement.close(); + connection.close(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f3d79b1df1594edc4fffd626244bc742ea13ec75 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java @@ -0,0 +1,187 @@ +package com.taosdata.jdbc.cases; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class InvalidResultSetPointerTest { + + private static String host = "127.0.0.1"; + private static final String dbName = "test"; + private static final String stbName = "stb"; + private static final String tbName = "tb"; + private static Connection connection; + private static int numOfSTb = 30000; + private static int numOfTb = 3; + private static int numOfThreads = 100; + + @Test + public void test() throws SQLException { + execute("drop database if exists " + dbName); + execute("create database if not exists " + dbName); + execute("use " + dbName); + createSTable(); + createTable(); + insert(); + selectMultiThreading(); + close(); + } + + private void insert() { + for (int i = 0; i < numOfSTb; i++) { + for (int j = 0; j < numOfTb; j++) { + final String sql = "INSERT INTO " + dbName + "." + tbName + i + "_" + j + " (ts, temperature, humidity, name) values(now, 20.5, 34, \"" + i + "\")"; + System.out.println(sql); + execute(sql); + } + } + } + + private void createSTable() { + for (int i = 0; i < numOfSTb; i++) { + final String sql = "create table if not exists " + dbName + "." + stbName + i + " (ts timestamp, temperature float, humidity int, name BINARY(" + (i % 73 + 10) + ")) TAGS (tag1 INT)"; + execute(sql); + } + } + + private void createTable() { + for (int i = 0; i < numOfSTb; i++) { + for (int j = 0; j < numOfTb; j++) { + final String sql = "create table if not exists " + dbName + "." + tbName + i + "_" + j + " USING " + stbName + i + " TAGS(" + j + ")"; + execute(sql); + } + } + } + + private void close() throws SQLException { + if (connection != null) { + this.connection.close(); + System.out.println("connection closed."); + } + } + + private void selectMultiThreading() { + int a = numOfSTb / numOfThreads; + if (a < 1) { + numOfThreads = numOfSTb; + a = 1; + } + + int b = 0; + if (numOfThreads != 0) { + b = numOfSTb % numOfThreads; + } + + multiThreadingClass instance[] = new multiThreadingClass[numOfThreads]; + + int last = 0; + for (int i = 0; i < numOfThreads; i++) { + instance[i] = new multiThreadingClass(); + instance[i].id = i; + instance[i].from = last; + if (i < b) { + instance[i].to = last + a; + } else { + instance[i].to = last + a - 1; + } + + last = instance[i].to + 1; + instance[i].numOfTb = numOfTb; + instance[i].connection = connection; + instance[i].dbName = dbName; + instance[i].tbName = tbName; + instance[i].start(); + } + + for (int i = 0; i < numOfThreads; i++) { + try { + instance[i].join(); + } catch (InterruptedException ie) { + ie.printStackTrace(); + } + } + } + + @BeforeClass + public static void beforeClass() { + try { + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty("charset", "UTF-8"); + properties.setProperty("locale", "en_US.UTF-8"); + properties.setProperty("timezone", "UTC-8"); + System.out.println("get connection starting..."); + connection = DriverManager.getConnection(url, properties); + if (connection != null) + System.out.println("[ OK ] Connection established."); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + private void execute(String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + statement.execute(sql); + long end = System.currentTimeMillis(); + printSql(sql, (end - start)); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private void printSql(String sql, long cost) { + System.out.println("time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private void executeQuery(String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + long end = System.currentTimeMillis(); + printSql(sql, (end - start)); +// printResult(resultSet); + resultSet.close(); + statement.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + class multiThreadingClass extends Thread { + public int id; + public int from, to; + public int numOfTb; + public Connection connection; + public String dbName; + public String tbName; + + public void run() { + System.out.println("ID: " + id + " from: " + from + " to: " + to); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + System.out.println("Thread " + id + " interrupted."); + } + + for (int i = from; i < to; i++) { + for (int j = 0; j < numOfTb; j++) { + if (j % 1000 == 0) { + try { + System.out.print(id + "s."); + Thread.sleep(1); + } catch (InterruptedException e) { + System.out.println("Thread " + id + " interrupted."); + } + } + final String sql = "select last_row(humidity) from " + dbName + "." + tbName + i + "_" + j; + executeQuery(sql); + } + } + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java new file mode 100644 index 0000000000000000000000000000000000000000..5cb76cc0cb71e1cb879b696d05cc6682f93d0bdc --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java @@ -0,0 +1,91 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.concurrent.TimeUnit; + +public class MultiThreadsWithSameStatmentTest { + + + private class Service { + public Connection conn; + public Statement stmt; + + public Service() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists jdbctest"); + stmt.executeUpdate("create table if not exists jdbctest.weather (ts timestamp, f1 int)"); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + public void release() { + try { + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + @Before + public void before() { + } + + @Test + public void test() { + Thread t1 = new Thread(() -> { + try { + Service service = new Service(); + ResultSet resultSet = service.stmt.executeQuery("select * from jdbctest.weather"); + while (resultSet.next()) { + ResultSetMetaData metaData = resultSet.getMetaData(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + System.out.print(metaData.getColumnLabel(i) + ": " + resultSet.getString(i)); + } + System.out.println(); + } + resultSet.close(); + service.release(); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + + Thread t2 = new Thread(() -> { + while (true) { + try { + Service service = new Service(); + service.stmt.executeUpdate("insert into jdbctest.weather values(now,1)"); + service.release(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + }); + t1.start(); + sleep(1000); + t2.start(); + } + + private void sleep(long mills) { + try { + TimeUnit.MILLISECONDS.sleep(mills); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d0ba113b7a4a8f99e22eb8143905d0b086583e1d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java @@ -0,0 +1,73 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class QueryDataTest { + + static Connection connection; + static Statement statement; + static String dbName = "test"; + static String stbName = "meters"; + static String host = "127.0.0.1"; + + @Before + public void createDatabase() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.executeUpdate("use " + dbName); + + String createTableSql = "create table " + stbName + "(ts timestamp, name binary(64))"; + statement.executeUpdate(createTableSql); + + } catch (ClassNotFoundException | SQLException e) { + return; + } + } + + @Test + public void testQueryBinaryData() throws SQLException { + String insertSql = "insert into " + stbName + " values(now, 'taosdata')"; + System.out.println(insertSql); + statement.executeUpdate(insertSql); + + String querySql = "select * from " + stbName; + ResultSet rs = statement.executeQuery(querySql); + + while (rs.next()) { + String name = rs.getString(2); + System.out.println("name = " + name); + assertEquals("taosdata", name); + } + rs.close(); + } + + @After + public void close() { + try { + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java new file mode 100644 index 0000000000000000000000000000000000000000..38c8cbb98c48342f131f4f5f0fee885bb446e83c --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java @@ -0,0 +1,82 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class SelectTest { + Connection connection; + String dbName = "test"; + String tName = "t0"; + String host = "127.0.0.1"; + + @Before + public void createDatabaseAndTable() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists " + dbName); + stmt.execute("create database if not exists " + dbName); + stmt.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); + stmt.close(); + } catch (ClassNotFoundException e) { + return; + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Test + public void selectData() { + long ts = 1496732686000l; + + try (Statement stmt = connection.createStatement()) { + for (int i = 0; i < 50; i++) { + ts++; + int row = stmt.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); + System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); + assertEquals(1, row); + } + + String sql = "select * from " + dbName + "." + tName; + ResultSet resSet = stmt.executeQuery(sql); + + int num = 0; + while (resSet.next()) { + num++; + } + resSet.close(); + assertEquals(num, 50); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @After + public void close() { + try { + if (connection != null) { + Statement stmt = connection.createStatement(); + stmt.executeUpdate("drop database " + dbName); + stmt.close(); + connection.close(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java new file mode 100644 index 0000000000000000000000000000000000000000..4575cb73a05fbbc19d6eaf2ba5be0ed27b61804c --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java @@ -0,0 +1,111 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class StableTest { + + private static Connection connection; + private static String dbName = "test"; + private static String stbName = "st"; + private static String host = "127.0.0.1"; + + @BeforeClass + public static void createDatabase() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + Statement statement = connection.createStatement(); + statement.execute("drop database if exists " + dbName); + statement.execute("create database if not exists " + dbName); + statement.execute("use " + dbName); + statement.close(); + } catch (ClassNotFoundException e) { + return; + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void case001_createSuperTable() { + try (Statement stmt = connection.createStatement()) { + final String sql = "create table " + stbName + " (ts timestamp, v1 int, v2 int) tags (tg nchar(20)) "; + stmt.execute(sql); + } catch (SQLException e) { + assert false : "error create stable" + e.getMessage(); + } + } + + @Test + public void case002_createTable() { + try (Statement stmt = connection.createStatement()) { + final String sql = "create table t1 using " + stbName + " tags (\"beijing\")"; + stmt.execute(sql); + } catch (SQLException e) { + assert false : "error create table" + e.getMessage(); + } + } + + @Test + public void case003_describeSTable() { + int num = 0; + try (Statement stmt = connection.createStatement()) { + String sql = "describe " + stbName; + ResultSet rs = stmt.executeQuery(sql); + while (rs.next()) { + for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) { + System.out.println(i + ":" + rs.getString(i)); + } + num++; + } + rs.close(); + assertEquals(4, num); + } catch (SQLException e) { + assert false : "error describe stable" + e.getMessage(); + } + } + + @Test + public void case004_describeTable() { + int num = 0; + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("describe t1"); + while (rs.next()) { + for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) { + System.out.printf("%d: %s\n", i, rs.getString(i)); + } + num++; + } + rs.close(); + assertEquals(4, num); + } catch (SQLException e) { + assert false : "error describe stable" + e.getMessage(); + } + } + + @AfterClass + public static void close() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..e9e36e20c4f83e98d6dd808da3a58ee628f418f0 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java @@ -0,0 +1,49 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Test; + +import java.sql.*; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class TaosInfoMonitorTest { + + @Test + public void testCreateTooManyConnection() throws ClassNotFoundException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + final String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"; + + List connectionList = IntStream.range(0, 100).mapToObj(i -> { + try { + TimeUnit.MILLISECONDS.sleep(100); + return DriverManager.getConnection(url); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + return null; + }).collect(Collectors.toList()); + + connectionList.stream().forEach(conn -> { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("show databases"); + while (rs.next()) { + + } + TimeUnit.MILLISECONDS.sleep(100); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + }); + + connectionList.stream().forEach(conn -> { + try { + conn.close(); + TimeUnit.MILLISECONDS.sleep(100); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + }); + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java deleted file mode 100644 index 0e2613d617074571c86c39ef253d68af9a0f6922..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.taosdata.jdbc.lib; - -import com.taosdata.jdbc.TSDBDriver; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; - -public class TSDBCommon { - - public static Connection getConn(String host) throws SQLException, ClassNotFoundException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - return DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - } - - public static void createDatabase(Connection connection, String dbName) throws SQLException { - Statement statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - statement.close(); - } - - public static void createStable(Connection connection, String stbName) throws SQLException { - Statement statement = connection.createStatement(); - String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; - statement.executeUpdate(createTableSql); - statement.close(); - } - - public static void createTables(Connection connection, int numOfTables, String stbName,String tablePrefix) throws SQLException { - Statement statement = connection.createStatement(); - for(int i = 0; i < numOfTables; i++) { - String loc = i % 2 == 0 ? "beijing" : "shanghai"; - String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; - statement.executeUpdate(createSubTalbesSql); - } - statement.close(); - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java new file mode 100644 index 0000000000000000000000000000000000000000..b0666989ba0e5aacfd6aa1110b4a57016a0937ee --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java @@ -0,0 +1,66 @@ +package com.taosdata.jdbc.rs; + +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; + +public class AuthenticationTest { + + private static final String host = "127.0.0.1"; + // private static final String host = "master"; + private static final String user = "root"; + private static final String password = "taos?data"; + private Connection conn; + + @Test + public void test() { + // change password + try { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass '" + password + "'"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + // use new to login and execute query + try { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); + Statement stmt = conn.createStatement(); + stmt.execute("show databases"); + ResultSet rs = stmt.getResultSet(); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + // change password back + try { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); + Statement stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass 'taosdata'"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Before + public void before() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java new file mode 100644 index 0000000000000000000000000000000000000000..4e005d129124ca480313f861a48af5fcfdbf25bc --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java @@ -0,0 +1,403 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class RestfulConnectionTest { + + private static final String host = "127.0.0.1"; + // private static final String host = "master"; + + private static Connection conn; + + @Test + public void getConnection() { + // already test in beforeClass method + } + + @Test + public void createStatement() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void prepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()"); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void prepareCall() throws SQLException { + conn.prepareCall("select server_status()"); + } + + @Test + public void nativeSQL() throws SQLException { + String nativeSQL = conn.nativeSQL("select * from log.log"); + Assert.assertEquals("select * from log.log", nativeSQL); + } + + @Test + public void setAutoCommit() throws SQLException { + conn.setAutoCommit(true); + conn.setAutoCommit(false); + } + + @Test + public void getAutoCommit() throws SQLException { + Assert.assertTrue(conn.getAutoCommit()); + } + + @Test + public void commit() throws SQLException { + conn.commit(); + } + + @Test + public void rollback() throws SQLException { + conn.rollback(); + } + + @Test + public void close() { + // connection will close in afterClass method + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(conn.isClosed()); + } + + @Test + public void getMetaData() throws SQLException { + DatabaseMetaData meta = conn.getMetaData(); + Assert.assertNotNull(meta); + Assert.assertEquals("com.taosdata.jdbc.rs.RestfulDriver", meta.getDriverName()); + } + + @Test + public void setReadOnly() throws SQLException { + conn.setReadOnly(true); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertTrue(conn.isReadOnly()); + } + + @Test + public void setCatalog() throws SQLException { + conn.setCatalog("test"); + Assert.assertEquals("test", conn.getCatalog()); + } + + @Test + public void getCatalog() throws SQLException { + conn.setCatalog("log"); + Assert.assertEquals("log", conn.getCatalog()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTransactionIsolation() throws SQLException { + conn.setTransactionIsolation(Connection.TRANSACTION_NONE); + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); + } + + @Test + public void getTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(conn.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + conn.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getTypeMap() throws SQLException { + conn.getTypeMap(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTypeMap() throws SQLException { + conn.setTypeMap(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setHoldability() throws SQLException { + conn.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + conn.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSavepoint() throws SQLException { + conn.setSavepoint(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testSetSavepoint() throws SQLException { + conn.setSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testRollback() throws SQLException { + conn.rollback(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void releaseSavepoint() throws SQLException { + conn.releaseSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement1() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement1() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall1() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement2() throws SQLException { + Assert.assertNotNull("", Statement.NO_GENERATED_KEYS); + conn.prepareStatement("", Statement.RETURN_GENERATED_KEYS); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement3() throws SQLException { + conn.prepareStatement("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement4() throws SQLException { + conn.prepareStatement("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createClob() throws SQLException { + conn.createClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createBlob() throws SQLException { + conn.createBlob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createNClob() throws SQLException { + conn.createNClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createSQLXML() throws SQLException { + conn.createSQLXML(); + } + + @Test(expected = SQLException.class) + public void isValid() throws SQLException { + Assert.assertTrue(conn.isValid(10)); + Assert.assertTrue(conn.isValid(0)); + conn.isValid(-1); + } + + @Test + public void setClientInfo() throws SQLClientInfoException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTC-8"); + } + + @Test + public void testSetClientInfo() throws SQLClientInfoException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn.setClientInfo(properties); + } + + @Test + public void getClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + Properties info = conn.getClientInfo(); + String charset = info.getProperty(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = info.getProperty(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = info.getProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test + public void testGetClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String charset = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createArrayOf() throws SQLException { + conn.createArrayOf("", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createStruct() throws SQLException { + conn.createStruct("", null); + } + + @Test + public void setSchema() throws SQLException { + conn.setSchema("test"); + } + + @Test + public void getSchema() throws SQLException { + Assert.assertNull(conn.getSchema()); + } + + @Test + public void abort() throws SQLException { + conn.abort(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNetworkTimeout() throws SQLException { + conn.setNetworkTimeout(null, 1000); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNetworkTimeout() throws SQLException { + conn.getNetworkTimeout(); + } + + @Test + public void unwrap() { + try { + RestfulConnection restfulConnection = conn.unwrap(RestfulConnection.class); + Assert.assertNotNull(restfulConnection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(conn.isWrapperFor(RestfulConnection.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/log?user=root&password=taosdata", properties); + // create test database for test cases + try (Statement stmt = conn.createStatement()) { + stmt.execute("create database if not exists test"); + } + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..1991c17065a34c16fe7758486bd10b83f3241a07 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java @@ -0,0 +1,984 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class RestfulDatabaseMetaDataTest { + // private static final String host = "master"; + private static final String host = "127.0.0.1"; + private static final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + private static Connection connection; + private static RestfulDatabaseMetaData metaData; + + @Test + public void unwrap() throws SQLException { + RestfulDatabaseMetaData unwrap = metaData.unwrap(RestfulDatabaseMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(metaData.isWrapperFor(RestfulDatabaseMetaData.class)); + } + + @Test + public void allProceduresAreCallable() throws SQLException { + Assert.assertFalse(metaData.allProceduresAreCallable()); + } + + @Test + public void allTablesAreSelectable() throws SQLException { + Assert.assertFalse(metaData.allTablesAreSelectable()); + } + + @Test + public void getURL() throws SQLException { + Assert.assertEquals(url, metaData.getURL()); + } + + @Test + public void getUserName() throws SQLException { + Assert.assertEquals("root", metaData.getUserName()); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertFalse(metaData.isReadOnly()); + } + + @Test + public void nullsAreSortedHigh() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedHigh()); + } + + @Test + public void nullsAreSortedLow() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedLow()); + } + + @Test + public void nullsAreSortedAtStart() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedAtStart()); + } + + @Test + public void nullsAreSortedAtEnd() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedAtEnd()); + } + + @Test + public void getDatabaseProductName() throws SQLException { + Assert.assertEquals("TDengine", metaData.getDatabaseProductName()); + } + + @Test + public void getDatabaseProductVersion() throws SQLException { + Assert.assertEquals("2.0.x.x", metaData.getDatabaseProductVersion()); + } + + @Test + public void getDriverName() throws SQLException { + Assert.assertEquals("com.taosdata.jdbc.rs.RestfulDriver", metaData.getDriverName()); + } + + @Test + public void getDriverVersion() throws SQLException { + Assert.assertEquals("2.0.x", metaData.getDriverVersion()); + } + + @Test + public void getDriverMajorVersion() { + Assert.assertEquals(2, metaData.getDriverMajorVersion()); + } + + @Test + public void getDriverMinorVersion() { + Assert.assertEquals(0, metaData.getDriverMinorVersion()); + } + + @Test + public void usesLocalFiles() throws SQLException { + Assert.assertFalse(metaData.usesLocalFiles()); + } + + @Test + public void usesLocalFilePerTable() throws SQLException { + Assert.assertFalse(metaData.usesLocalFilePerTable()); + } + + @Test + public void supportsMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseIdentifiers()); + } + + @Test + public void storesUpperCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseIdentifiers()); + } + + @Test + public void storesLowerCaseIdentifiers() throws SQLException { + Assert.assertTrue(metaData.storesLowerCaseIdentifiers()); + } + + @Test + public void storesMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseIdentifiers()); + } + + @Test + public void supportsMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseQuotedIdentifiers()); + } + + @Test + public void storesUpperCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseQuotedIdentifiers()); + } + + @Test + public void storesLowerCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesLowerCaseQuotedIdentifiers()); + } + + @Test + public void storesMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseQuotedIdentifiers()); + } + + @Test + public void getIdentifierQuoteString() throws SQLException { + Assert.assertEquals(" ", metaData.getIdentifierQuoteString()); + } + + @Test + public void getSQLKeywords() throws SQLException { + Assert.assertEquals(null, metaData.getSQLKeywords()); + } + + @Test + public void getNumericFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getNumericFunctions()); + } + + @Test + public void getStringFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getStringFunctions()); + } + + @Test + public void getSystemFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getSystemFunctions()); + } + + @Test + public void getTimeDateFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getTimeDateFunctions()); + } + + @Test + public void getSearchStringEscape() throws SQLException { + Assert.assertEquals(null, metaData.getSearchStringEscape()); + } + + @Test + public void getExtraNameCharacters() throws SQLException { + Assert.assertEquals(null, metaData.getExtraNameCharacters()); + } + + @Test + public void supportsAlterTableWithAddColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithAddColumn()); + } + + @Test + public void supportsAlterTableWithDropColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithDropColumn()); + } + + @Test + public void supportsColumnAliasing() throws SQLException { + Assert.assertTrue(metaData.supportsColumnAliasing()); + } + + @Test + public void nullPlusNonNullIsNull() throws SQLException { + Assert.assertFalse(metaData.nullPlusNonNullIsNull()); + } + + @Test + public void supportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert()); + } + + @Test + public void testSupportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert(1, 1)); + } + + @Test + public void supportsTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsTableCorrelationNames()); + } + + @Test + public void supportsDifferentTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsDifferentTableCorrelationNames()); + } + + @Test + public void supportsExpressionsInOrderBy() throws SQLException { + Assert.assertFalse(metaData.supportsExpressionsInOrderBy()); + } + + @Test + public void supportsOrderByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsOrderByUnrelated()); + } + + @Test + public void supportsGroupBy() throws SQLException { + Assert.assertTrue(metaData.supportsGroupBy()); + } + + @Test + public void supportsGroupByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByUnrelated()); + } + + @Test + public void supportsGroupByBeyondSelect() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByBeyondSelect()); + } + + @Test + public void supportsLikeEscapeClause() throws SQLException { + Assert.assertFalse(metaData.supportsLikeEscapeClause()); + } + + @Test + public void supportsMultipleResultSets() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleResultSets()); + } + + @Test + public void supportsMultipleTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleTransactions()); + } + + @Test + public void supportsNonNullableColumns() throws SQLException { + Assert.assertFalse(metaData.supportsNonNullableColumns()); + } + + @Test + public void supportsMinimumSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsMinimumSQLGrammar()); + } + + @Test + public void supportsCoreSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsCoreSQLGrammar()); + } + + @Test + public void supportsExtendedSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsExtendedSQLGrammar()); + } + + @Test + public void supportsANSI92EntryLevelSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92EntryLevelSQL()); + } + + @Test + public void supportsANSI92IntermediateSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92IntermediateSQL()); + } + + @Test + public void supportsANSI92FullSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92FullSQL()); + } + + @Test + public void supportsIntegrityEnhancementFacility() throws SQLException { + Assert.assertFalse(metaData.supportsIntegrityEnhancementFacility()); + } + + @Test + public void supportsOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsOuterJoins()); + } + + @Test + public void supportsFullOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsFullOuterJoins()); + } + + @Test + public void supportsLimitedOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsLimitedOuterJoins()); + } + + @Test + public void getSchemaTerm() throws SQLException { + Assert.assertNull(metaData.getSchemaTerm()); + } + + @Test + public void getProcedureTerm() throws SQLException { + Assert.assertNull(metaData.getProcedureTerm()); + } + + @Test + public void getCatalogTerm() throws SQLException { + Assert.assertEquals("database", metaData.getCatalogTerm()); + } + + @Test + public void isCatalogAtStart() throws SQLException { + Assert.assertTrue(metaData.isCatalogAtStart()); + } + + @Test + public void getCatalogSeparator() throws SQLException { + Assert.assertEquals(".", metaData.getCatalogSeparator()); + } + + @Test + public void supportsSchemasInDataManipulation() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInDataManipulation()); + } + + @Test + public void supportsSchemasInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInProcedureCalls()); + } + + @Test + public void supportsSchemasInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInTableDefinitions()); + } + + @Test + public void supportsSchemasInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInIndexDefinitions()); + } + + @Test + public void supportsSchemasInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInPrivilegeDefinitions()); + } + + @Test + public void supportsCatalogsInDataManipulation() throws SQLException { + Assert.assertTrue(metaData.supportsCatalogsInDataManipulation()); + } + + @Test + public void supportsCatalogsInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInProcedureCalls()); + } + + @Test + public void supportsCatalogsInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInTableDefinitions()); + } + + @Test + public void supportsCatalogsInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInIndexDefinitions()); + } + + @Test + public void supportsCatalogsInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInPrivilegeDefinitions()); + } + + @Test + public void supportsPositionedDelete() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedDelete()); + } + + @Test + public void supportsPositionedUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedUpdate()); + } + + @Test + public void supportsSelectForUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsSelectForUpdate()); + } + + @Test + public void supportsStoredProcedures() throws SQLException { + Assert.assertFalse(metaData.supportsStoredProcedures()); + } + + @Test + public void supportsSubqueriesInComparisons() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInComparisons()); + } + + @Test + public void supportsSubqueriesInExists() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInExists()); + } + + @Test + public void supportsSubqueriesInIns() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInIns()); + } + + @Test + public void supportsSubqueriesInQuantifieds() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInQuantifieds()); + } + + @Test + public void supportsCorrelatedSubqueries() throws SQLException { + Assert.assertFalse(metaData.supportsCorrelatedSubqueries()); + } + + @Test + public void supportsUnion() throws SQLException { + Assert.assertFalse(metaData.supportsUnion()); + } + + @Test + public void supportsUnionAll() throws SQLException { + Assert.assertFalse(metaData.supportsUnionAll()); + } + + @Test + public void supportsOpenCursorsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossCommit()); + } + + @Test + public void supportsOpenCursorsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossRollback()); + } + + @Test + public void supportsOpenStatementsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossCommit()); + } + + @Test + public void supportsOpenStatementsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossRollback()); + } + + @Test + public void getMaxBinaryLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxBinaryLiteralLength()); + } + + @Test + public void getMaxCharLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCharLiteralLength()); + } + + @Test + public void getMaxColumnNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnNameLength()); + } + + @Test + public void getMaxColumnsInGroupBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInGroupBy()); + } + + @Test + public void getMaxColumnsInIndex() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInIndex()); + } + + @Test + public void getMaxColumnsInOrderBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInOrderBy()); + } + + @Test + public void getMaxColumnsInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInSelect()); + } + + @Test + public void getMaxColumnsInTable() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInTable()); + } + + @Test + public void getMaxConnections() throws SQLException { + Assert.assertEquals(0, metaData.getMaxConnections()); + } + + @Test + public void getMaxCursorNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCursorNameLength()); + } + + @Test + public void getMaxIndexLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxIndexLength()); + } + + @Test + public void getMaxSchemaNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxSchemaNameLength()); + } + + @Test + public void getMaxProcedureNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxProcedureNameLength()); + } + + @Test + public void getMaxCatalogNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCatalogNameLength()); + } + + @Test + public void getMaxRowSize() throws SQLException { + Assert.assertEquals(0, metaData.getMaxRowSize()); + } + + @Test + public void doesMaxRowSizeIncludeBlobs() throws SQLException { + Assert.assertFalse(metaData.doesMaxRowSizeIncludeBlobs()); + } + + @Test + public void getMaxStatementLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatementLength()); + } + + @Test + public void getMaxStatements() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatements()); + } + + @Test + public void getMaxTableNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTableNameLength()); + } + + @Test + public void getMaxTablesInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTablesInSelect()); + } + + @Test + public void getMaxUserNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxUserNameLength()); + } + + @Test + public void getDefaultTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, metaData.getDefaultTransactionIsolation()); + } + + @Test + public void supportsTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsTransactions()); + } + + @Test + public void supportsTransactionIsolationLevel() throws SQLException { + Assert.assertTrue(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_NONE)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_COMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_UNCOMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_REPEATABLE_READ)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_SERIALIZABLE)); + } + + @Test + public void supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsDataDefinitionAndDataManipulationTransactions()); + } + + @Test + public void supportsDataManipulationTransactionsOnly() throws SQLException { + Assert.assertFalse(metaData.supportsDataManipulationTransactionsOnly()); + } + + @Test + public void dataDefinitionCausesTransactionCommit() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionCausesTransactionCommit()); + } + + @Test + public void dataDefinitionIgnoredInTransactions() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionIgnoredInTransactions()); + } + + @Test + public void getProcedures() throws SQLException { + Assert.assertNull(metaData.getProcedures("*", "*", "*")); + } + + @Test + public void getProcedureColumns() throws SQLException { + Assert.assertNull(metaData.getProcedureColumns("*", "*", "*", "*")); + } + + @Test + public void getTables() throws SQLException { + System.out.println("****************************************************"); + ResultSet tables = metaData.getTables("log", "", null, null); + ResultSetMetaData metaData = tables.getMetaData(); + while (tables.next()) { + System.out.print(metaData.getColumnLabel(1) + ":" + tables.getString(1) + "\t"); + System.out.print(metaData.getColumnLabel(3) + ":" + tables.getString(3) + "\t"); + System.out.print(metaData.getColumnLabel(4) + ":" + tables.getString(4) + "\t"); + System.out.print(metaData.getColumnLabel(5) + ":" + tables.getString(5) + "\n"); + } + System.out.println(); + Assert.assertNotNull(tables); + } + + @Test + public void getSchemas() throws SQLException { + Assert.assertNotNull(metaData.getSchemas()); + } + + @Test + public void getCatalogs() throws SQLException { + System.out.println("****************************************************"); + + ResultSet catalogs = metaData.getCatalogs(); + ResultSetMetaData meta = catalogs.getMetaData(); + while (catalogs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + catalogs.getString(i)); + } + System.out.println(); + } + } + + @Test + public void getTableTypes() throws SQLException { + System.out.println("****************************************************"); + + ResultSet tableTypes = metaData.getTableTypes(); + while (tableTypes.next()) { + System.out.println(tableTypes.getString("TABLE_TYPE")); + } + Assert.assertNotNull(metaData.getTableTypes()); + } + + @Test + public void getColumns() throws SQLException { + System.out.println("****************************************************"); + + ResultSet columns = metaData.getColumns("log", "", "dn", ""); + ResultSetMetaData meta = columns.getMetaData(); + while (columns.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + columns.getString(1) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + columns.getString(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + columns.getString(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + columns.getString(5) + "\t"); + System.out.print(meta.getColumnLabel(6) + ": " + columns.getString(6) + "\t"); + System.out.print(meta.getColumnLabel(7) + ": " + columns.getString(7) + "\t"); + System.out.print(meta.getColumnLabel(9) + ": " + columns.getString(9) + "\t"); + System.out.print(meta.getColumnLabel(10) + ": " + columns.getString(10) + "\t"); + System.out.print(meta.getColumnLabel(11) + ": " + columns.getString(11) + "\n"); + System.out.print(meta.getColumnLabel(12) + ": " + columns.getString(12) + "\n"); + } + } + + @Test + public void getColumnPrivileges() throws SQLException { + Assert.assertNotNull(metaData.getColumnPrivileges("", "", "", "")); + } + + @Test + public void getTablePrivileges() throws SQLException { + Assert.assertNotNull(metaData.getTablePrivileges("", "", "")); + } + + @Test + public void getBestRowIdentifier() throws SQLException { + Assert.assertNotNull(metaData.getBestRowIdentifier("", "", "", 0, false)); + } + + @Test + public void getVersionColumns() throws SQLException { + Assert.assertNotNull(metaData.getVersionColumns("", "", "")); + } + + @Test + public void getPrimaryKeys() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getPrimaryKeys("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("COLUMN_NAME: " + rs.getString("COLUMN_NAME")); + System.out.println("KEY_SEQ: " + rs.getString("KEY_SEQ")); + System.out.println("PK_NAME: " + rs.getString("PK_NAME")); + } + + Assert.assertNotNull(rs); + } + + @Test + public void getImportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getImportedKeys("", "", "")); + } + + @Test + public void getExportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getExportedKeys("", "", "")); + } + + @Test + public void getCrossReference() throws SQLException { + Assert.assertNotNull(metaData.getCrossReference("", "", "", "", "", "")); + } + + @Test + public void getTypeInfo() throws SQLException { + Assert.assertNotNull(metaData.getTypeInfo()); + } + + @Test + public void getIndexInfo() throws SQLException { + Assert.assertNotNull(metaData.getIndexInfo("", "", "", false, false)); + } + + @Test + public void supportsResultSetType() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetType(0)); + } + + @Test + public void supportsResultSetConcurrency() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetConcurrency(0, 0)); + } + + @Test + public void ownUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownUpdatesAreVisible(0)); + } + + @Test + public void ownDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownDeletesAreVisible(0)); + } + + @Test + public void ownInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownInsertsAreVisible(0)); + } + + @Test + public void othersUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersUpdatesAreVisible(0)); + } + + @Test + public void othersDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersDeletesAreVisible(0)); + } + + @Test + public void othersInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersInsertsAreVisible(0)); + } + + @Test + public void updatesAreDetected() throws SQLException { + Assert.assertFalse(metaData.updatesAreDetected(0)); + } + + @Test + public void deletesAreDetected() throws SQLException { + Assert.assertFalse(metaData.deletesAreDetected(0)); + } + + @Test + public void insertsAreDetected() throws SQLException { + Assert.assertFalse(metaData.insertsAreDetected(0)); + } + + @Test + public void supportsBatchUpdates() throws SQLException { + Assert.assertFalse(metaData.supportsBatchUpdates()); + } + + @Test + public void getUDTs() throws SQLException { + Assert.assertNotNull(metaData.getUDTs("", "", "", null)); + } + + @Test + public void getConnection() throws SQLException { + Assert.assertNotNull(metaData.getConnection()); + } + + @Test + public void supportsSavepoints() throws SQLException { + Assert.assertFalse(metaData.supportsSavepoints()); + } + + @Test + public void supportsNamedParameters() throws SQLException { + Assert.assertFalse(metaData.supportsNamedParameters()); + } + + @Test + public void supportsMultipleOpenResults() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleOpenResults()); + } + + @Test + public void supportsGetGeneratedKeys() throws SQLException { + Assert.assertFalse(metaData.supportsGetGeneratedKeys()); + } + + @Test + public void getSuperTypes() throws SQLException { + Assert.assertNotNull(metaData.getSuperTypes("", "", "")); + } + + @Test + public void getSuperTables() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getSuperTables("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("SUPERTABLE_NAME: " + rs.getString("SUPERTABLE_NAME")); + } + Assert.assertNotNull(rs); + } + + @Test + public void getAttributes() throws SQLException { + Assert.assertNotNull(metaData.getAttributes("", "", "", "")); + } + + @Test + public void supportsResultSetHoldability() throws SQLException { + Assert.assertTrue(metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT)); + Assert.assertFalse(metaData.supportsResultSetHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT)); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(1, metaData.getResultSetHoldability()); + } + + @Test + public void getDatabaseMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getDatabaseMajorVersion()); + } + + @Test + public void getDatabaseMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getDatabaseMinorVersion()); + } + + @Test + public void getJDBCMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getJDBCMajorVersion()); + } + + @Test + public void getJDBCMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getJDBCMinorVersion()); + } + + @Test + public void getSQLStateType() throws SQLException { + Assert.assertEquals(0, metaData.getSQLStateType()); + } + + @Test + public void locatorsUpdateCopy() throws SQLException { + Assert.assertFalse(metaData.locatorsUpdateCopy()); + } + + @Test + public void supportsStatementPooling() throws SQLException { + Assert.assertFalse(metaData.supportsStatementPooling()); + } + + @Test + public void getRowIdLifetime() throws SQLException { + Assert.assertNull(metaData.getRowIdLifetime()); + } + + @Test + public void supportsStoredFunctionsUsingCallSyntax() throws SQLException { + Assert.assertFalse(metaData.supportsStoredFunctionsUsingCallSyntax()); + } + + @Test + public void autoCommitFailureClosesAllResultSets() throws SQLException { + Assert.assertFalse(metaData.autoCommitFailureClosesAllResultSets()); + } + + @Test + public void getClientInfoProperties() throws SQLException { + Assert.assertNotNull(metaData.getClientInfoProperties()); + } + + @Test + public void getFunctions() throws SQLException { + Assert.assertNotNull(metaData.getFunctions("", "", "")); + } + + @Test + public void getFunctionColumns() throws SQLException { + Assert.assertNotNull(metaData.getFunctionColumns("", "", "", "")); + } + + @Test + public void getPseudoColumns() throws SQLException { + Assert.assertNotNull(metaData.getPseudoColumns("", "", "", "")); + } + + @Test + public void generatedKeyAlwaysReturned() throws SQLException { + Assert.assertFalse(metaData.generatedKeyAlwaysReturned()); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection(url, properties); + metaData = connection.getMetaData().unwrap(RestfulDatabaseMetaData.class); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index d07a6a21796fc2ea7927c04fd0cf5a37e9a49988..e239aa068bae0ee70d204aec4412bf29e5b36bf1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.sql.*; public class RestfulDriverTest { + private static final String host = "127.0.0.1"; @Test public void connect() { @@ -15,9 +16,9 @@ public class RestfulDriverTest { @Test public void acceptsURL() throws SQLException { Driver driver = new RestfulDriver(); - boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://master:6041"); + boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://" + host + ":6041"); Assert.assertTrue(isAccept); - isAccept = driver.acceptsURL("jdbc:TAOS://master:6041"); + isAccept = driver.acceptsURL("jdbc:TAOS://" + host + ":6041"); Assert.assertFalse(isAccept); } @@ -26,6 +27,9 @@ public class RestfulDriverTest { Driver driver = new RestfulDriver(); final String url = ""; DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null); + for (DriverPropertyInfo prop : propertyInfo) { + System.out.println(prop); + } } @Test diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index d13475b96dfc96780faaae7b01275f717f8befbf..451f5d49160a69b4786e806c51480236083eacac 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -9,23 +9,13 @@ import java.util.Random; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class RestfulJDBCTest { - private Connection connection; - - @Before - public void before() throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/restful_test?user=root&password=taosdata"); - } - - @After - public void after() throws SQLException { - if (connection != null) - connection.close(); - } - + private static final String host = "127.0.0.1"; +// private static final String host = "master"; + private static Connection connection; + private Random random = new Random(System.currentTimeMillis()); /** - * 查询所有log.log + * select * from log.log **/ @Test public void testCase001() { @@ -84,7 +74,6 @@ public class RestfulJDBCTest { } } - private Random random = new Random(System.currentTimeMillis()); @Test public void testCase005() { @@ -104,5 +93,50 @@ public class RestfulJDBCTest { } } + @Test + public void testCase006() { + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + while (rs.next()) { + System.out.print("ts: " + rs.getTimestamp("ts")); + System.out.print(", temperature: " + rs.getString("temperature")); + System.out.print(", humidity: " + rs.getString("humidity")); + System.out.println(", location: " + rs.getString("location")); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase007() { + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + int columnCount = meta.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + String columnLabel = meta.getColumnLabel(i); + String value = rs.getString(i); + System.out.print(columnLabel + ": " + value + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void before() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + } + + @AfterClass + public static void after() throws SQLException { + if (connection != null) + connection.close(); + } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a3867c1b6e30d810dcea3001ee92eae698256341 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -0,0 +1,246 @@ +package com.taosdata.jdbc.rs; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class RestfulPreparedStatementTest { + private static final String host = "127.0.0.1"; + // private static final String host = "master"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; + private static PreparedStatement pstmt_select; + + @Test + public void executeQuery() throws SQLException { + long end = System.currentTimeMillis(); + long start = end - 1000 * 60 * 60; + pstmt_select.setTimestamp(1, new Timestamp(start)); + pstmt_select.setTimestamp(2, new Timestamp(end)); + pstmt_select.setInt(3, 0); + + ResultSet rs = pstmt_select.executeQuery(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + } + + @Test + public void executeUpdate() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(2, 3.14f); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + } + + @Test + public void setNull() throws SQLException { + pstmt_insert.setNull(2, Types.FLOAT); + } + + @Test + public void setBoolean() throws SQLException { + pstmt_insert.setBoolean(2, true); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setByte() throws SQLException { + pstmt_insert.setByte(1, (byte) 0x001); + } + + @Test + public void setShort() { + + } + + @Test + public void setInt() { + + } + + @Test + public void setLong() { + + } + + @Test + public void setFloat() { + + } + + @Test + public void setDouble() { + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBigDecimal() throws SQLException { + pstmt_insert.setBigDecimal(1, null); + } + + @Test + public void setString() { + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBytes() throws SQLException { + pstmt_insert.setBytes(1, new byte[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setDate() throws SQLException { + pstmt_insert.setDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTime() throws SQLException { + pstmt_insert.setTime(1, new Time(System.currentTimeMillis())); + } + + @Test + public void setTimestamp() { + //TODO + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setAsciiStream() throws SQLException { + pstmt_insert.setAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBinaryStream() throws SQLException { + pstmt_insert.setBinaryStream(1, null); + } + + @Test + public void clearParameters() { + //TODO + } + + @Test + public void setObject() throws SQLException { + pstmt_insert.setObject(1, System.currentTimeMillis()); + //TODO + } + + @Test + public void execute() { + //TODO + } + + @Test + public void addBatch() { + //TODO: + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCharacterStream() throws SQLException { + pstmt_insert.setCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRef() throws SQLException { + pstmt_insert.setRef(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBlob() throws SQLException { + pstmt_insert.setBlob(1, (Blob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setClob() throws SQLException { + pstmt_insert.setClob(1, (Clob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setArray() throws SQLException { + pstmt_insert.setArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getMetaData() throws SQLException { + pstmt_insert.getMetaData(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setURL() throws SQLException { + pstmt_insert.setURL(1, null); + } + + @Test + public void getParameterMetaData() throws SQLException { + ParameterMetaData parameterMetaData = pstmt_insert.getParameterMetaData(); + Assert.assertNull(parameterMetaData); + //TODO: + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRowId() throws SQLException { + pstmt_insert.setRowId(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNString() throws SQLException { + pstmt_insert.setNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNCharacterStream() throws SQLException { + pstmt_insert.setNCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNClob() throws SQLException { + pstmt_insert.setNClob(1, (NClob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSQLXML() throws SQLException { + pstmt_insert.setSQLXML(1, null); + } + + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + pstmt_select = conn.prepareStatement(sql_select); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..13973d8b6bf6c11ced27866724b91c0746a024d0 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -0,0 +1,221 @@ +package com.taosdata.jdbc.rs; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class RestfulResultSetMetaDataTest { + + private static final String host = "127.0.0.1"; +// private static final String host = "master"; + + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + private static ResultSetMetaData meta; + + @Test + public void getColumnCount() throws SQLException { + Assert.assertEquals(10, meta.getColumnCount()); + } + + @Test + public void isAutoIncrement() throws SQLException { + Assert.assertFalse(meta.isAutoIncrement(1)); + Assert.assertFalse(meta.isAutoIncrement(2)); + Assert.assertFalse(meta.isAutoIncrement(3)); + Assert.assertFalse(meta.isAutoIncrement(4)); + Assert.assertFalse(meta.isAutoIncrement(5)); + Assert.assertFalse(meta.isAutoIncrement(6)); + Assert.assertFalse(meta.isAutoIncrement(7)); + Assert.assertFalse(meta.isAutoIncrement(8)); + Assert.assertFalse(meta.isAutoIncrement(9)); + Assert.assertFalse(meta.isAutoIncrement(10)); + } + + @Test + public void isCaseSensitive() throws SQLException { + Assert.assertFalse(meta.isCaseSensitive(1)); + } + + @Test + public void isSearchable() throws SQLException { + Assert.assertTrue(meta.isSearchable(1)); + } + + @Test + public void isCurrency() throws SQLException { + Assert.assertFalse(meta.isCurrency(1)); + } + + @Test + public void isNullable() throws SQLException { + Assert.assertEquals(ResultSetMetaData.columnNoNulls, meta.isNullable(1)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(2)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(3)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(4)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(5)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(6)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(7)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(8)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(9)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(10)); + } + + @Test + public void isSigned() throws SQLException { + Assert.assertFalse(meta.isSigned(1)); + } + + @Test + public void getColumnDisplaySize() throws SQLException { + Assert.assertEquals(64, meta.getColumnDisplaySize(10)); + } + + @Test + public void getColumnLabel() throws SQLException { + Assert.assertEquals("f1", meta.getColumnLabel(1)); + } + + @Test + public void getColumnName() throws SQLException { + Assert.assertEquals("f1", meta.getColumnName(1)); + } + + @Test + public void getSchemaName() throws SQLException { + Assert.assertEquals("", meta.getSchemaName(1)); + } + + @Test + public void getPrecision() throws SQLException { + Assert.assertEquals(0, meta.getPrecision(1)); + } + + @Test + public void getScale() throws SQLException { + Assert.assertEquals(0, meta.getScale(1)); + } + + @Test + public void getTableName() throws SQLException { + Assert.assertEquals("", meta.getTableName(1)); + } + + @Test + public void getCatalogName() throws SQLException { + Assert.assertEquals("restful_test", meta.getCatalogName(1)); + Assert.assertEquals("restful_test", meta.getCatalogName(2)); + Assert.assertEquals("restful_test", meta.getCatalogName(3)); + Assert.assertEquals("restful_test", meta.getCatalogName(4)); + Assert.assertEquals("restful_test", meta.getCatalogName(5)); + Assert.assertEquals("restful_test", meta.getCatalogName(6)); + Assert.assertEquals("restful_test", meta.getCatalogName(7)); + Assert.assertEquals("restful_test", meta.getCatalogName(8)); + Assert.assertEquals("restful_test", meta.getCatalogName(9)); + Assert.assertEquals("restful_test", meta.getCatalogName(10)); + } + + @Test + public void getColumnType() throws SQLException { + Assert.assertEquals(Types.TIMESTAMP, meta.getColumnType(1)); + Assert.assertEquals(Types.INTEGER, meta.getColumnType(2)); + Assert.assertEquals(Types.BIGINT, meta.getColumnType(3)); + Assert.assertEquals(Types.FLOAT, meta.getColumnType(4)); + Assert.assertEquals(Types.DOUBLE, meta.getColumnType(5)); + Assert.assertEquals(Types.BINARY, meta.getColumnType(6)); + Assert.assertEquals(Types.SMALLINT, meta.getColumnType(7)); + Assert.assertEquals(Types.TINYINT, meta.getColumnType(8)); + Assert.assertEquals(Types.BOOLEAN, meta.getColumnType(9)); + Assert.assertEquals(Types.NCHAR, meta.getColumnType(10)); + } + + @Test + public void getColumnTypeName() throws SQLException { + Assert.assertEquals("TIMESTAMP", meta.getColumnTypeName(1)); + Assert.assertEquals("INT", meta.getColumnTypeName(2)); + Assert.assertEquals("BIGINT", meta.getColumnTypeName(3)); + Assert.assertEquals("FLOAT", meta.getColumnTypeName(4)); + Assert.assertEquals("DOUBLE", meta.getColumnTypeName(5)); + Assert.assertEquals("BINARY", meta.getColumnTypeName(6)); + Assert.assertEquals("SMALLINT", meta.getColumnTypeName(7)); + Assert.assertEquals("TINYINT", meta.getColumnTypeName(8)); + Assert.assertEquals("BOOL", meta.getColumnTypeName(9)); + Assert.assertEquals("NCHAR", meta.getColumnTypeName(10)); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertTrue(meta.isReadOnly(1)); + } + + @Test + public void isWritable() throws SQLException { + Assert.assertFalse(meta.isWritable(1)); + } + + @Test + public void isDefinitelyWritable() throws SQLException { + Assert.assertFalse(meta.isDefinitelyWritable(1)); + } + + @Test + public void getColumnClassName() throws SQLException { + Assert.assertEquals(Timestamp.class.getName(), meta.getColumnClassName(1)); + Assert.assertEquals(Integer.class.getName(), meta.getColumnClassName(2)); + Assert.assertEquals(Long.class.getName(), meta.getColumnClassName(3)); + Assert.assertEquals(Float.class.getName(), meta.getColumnClassName(4)); + Assert.assertEquals(Double.class.getName(), meta.getColumnClassName(5)); + Assert.assertEquals(String.class.getName(), meta.getColumnClassName(6)); + Assert.assertEquals(Short.class.getName(), meta.getColumnClassName(7)); + Assert.assertEquals(Short.class.getName(), meta.getColumnClassName(8)); + Assert.assertEquals(Boolean.class.getName(), meta.getColumnClassName(9)); + Assert.assertEquals(String.class.getName(), meta.getColumnClassName(10)); + } + + @Test + public void unwrap() throws SQLException { + Assert.assertNotNull(meta.unwrap(RestfulResultSetMetaData.class)); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(meta.isWrapperFor(RestfulResultSetMetaData.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + meta = rs.getMetaData(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java new file mode 100644 index 0000000000000000000000000000000000000000..b199eff1baab53d9c4a8c65f7e0bb58157657d33 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -0,0 +1,594 @@ +package com.taosdata.jdbc.rs; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.*; + +public class RestfulResultSetTest { + + private static final String host = "127.0.0.1"; +// private static final String host = "master"; + + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + + @Test + public void wasNull() throws SQLException { + Assert.assertFalse(rs.wasNull()); + } + + @Test + public void getString() throws SQLException { + String f10 = rs.getString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test + public void getBoolean() throws SQLException { + Boolean f9 = rs.getBoolean("f9"); + Assert.assertEquals(true, f9); + f9 = rs.getBoolean(9); + Assert.assertEquals(true, f9); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getByte() throws SQLException { + rs.getByte(1); + } + + @Test + public void getShort() throws SQLException { + short f7 = rs.getShort("f7"); + Assert.assertEquals(10, f7); + f7 = rs.getShort(7); + Assert.assertEquals(10, f7); + } + + @Test + public void getInt() throws SQLException { + int f2 = rs.getInt("f2"); + Assert.assertEquals(1, f2); + f2 = rs.getInt(2); + Assert.assertEquals(1, f2); + } + + @Test + public void getLong() throws SQLException { + long f3 = rs.getLong("f3"); + Assert.assertEquals(100, f3); + f3 = rs.getLong(3); + Assert.assertEquals(100, f3); + } + + @Test + public void getFloat() throws SQLException { + float f4 = rs.getFloat("f4"); + Assert.assertEquals(3.1415f, f4, 0f); + f4 = rs.getFloat(4); + Assert.assertEquals(3.1415f, f4, 0f); + } + + @Test + public void getDouble() throws SQLException { + double f5 = rs.getDouble("f5"); + Assert.assertEquals(3.1415926, f5, 0.0); + f5 = rs.getDouble(5); + Assert.assertEquals(3.1415926, f5, 0.0); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBigDecimal() throws SQLException { + rs.getBigDecimal("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBytes() throws SQLException { + rs.getBytes("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getDate() throws SQLException { + rs.getDate("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getTime() throws SQLException { + rs.getTime("f1"); + } + + @Test + public void getTimestamp() throws SQLException { + Timestamp f1 = rs.getTimestamp("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + f1 = rs.getTimestamp(1); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getAsciiStream() throws SQLException { + rs.getAsciiStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getUnicodeStream() throws SQLException { + rs.getUnicodeStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBinaryStream() throws SQLException { + rs.getBinaryStream("f1"); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(rs.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + rs.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCursorName() throws SQLException { + rs.getCursorName(); + } + + @Test + public void getMetaData() throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertNotNull(meta); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getObject() throws SQLException { + rs.getObject("f1"); + } + + @Test(expected = SQLException.class) + public void findColumn() throws SQLException { + int columnIndex = rs.findColumn("f1"); + Assert.assertEquals(1, columnIndex); + columnIndex = rs.findColumn("f2"); + Assert.assertEquals(2, columnIndex); + columnIndex = rs.findColumn("f3"); + Assert.assertEquals(3, columnIndex); + columnIndex = rs.findColumn("f4"); + Assert.assertEquals(4, columnIndex); + columnIndex = rs.findColumn("f5"); + Assert.assertEquals(5, columnIndex); + columnIndex = rs.findColumn("f6"); + Assert.assertEquals(6, columnIndex); + columnIndex = rs.findColumn("f7"); + Assert.assertEquals(7, columnIndex); + columnIndex = rs.findColumn("f8"); + Assert.assertEquals(8, columnIndex); + columnIndex = rs.findColumn("f9"); + Assert.assertEquals(9, columnIndex); + columnIndex = rs.findColumn("f10"); + Assert.assertEquals(10, columnIndex); + + rs.findColumn("f11"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCharacterStream() throws SQLException { + rs.getCharacterStream(1); + } + + @Test + public void isBeforeFirst() throws SQLException { + Assert.assertFalse(rs.isBeforeFirst()); + rs.beforeFirst(); + Assert.assertTrue(rs.isBeforeFirst()); + rs.next(); + } + + @Test + public void isAfterLast() throws SQLException { + Assert.assertFalse(rs.isAfterLast()); + } + + @Test + public void isFirst() throws SQLException { + Assert.assertTrue(rs.isFirst()); + } + + @Test + public void isLast() throws SQLException { + Assert.assertTrue(rs.isLast()); + } + + @Test + public void beforeFirst() throws SQLException { + rs.beforeFirst(); + Assert.assertTrue(rs.isBeforeFirst()); + rs.next(); + } + + @Test + public void afterLast() throws SQLException { + rs.afterLast(); + Assert.assertTrue(rs.isAfterLast()); + rs.first(); + } + + @Test + public void first() throws SQLException { + rs.first(); + Assert.assertEquals("2021-01-01 00:00:00.0", rs.getTimestamp("f1").toString()); + } + + @Test + public void last() throws SQLException { + rs.last(); + Assert.assertEquals("2021-01-01 00:00:00.0", rs.getTimestamp("f1").toString()); + } + + @Test + public void getRow() throws SQLException { + int row = rs.getRow(); + Assert.assertEquals(1, row); + rs.beforeFirst(); + row = rs.getRow(); + Assert.assertEquals(0, row); + rs.first(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void absolute() throws SQLException { + rs.absolute(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void relative() throws SQLException { + rs.relative(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void previous() throws SQLException { + rs.previous(); + } + + @Test + public void setFetchDirection() throws SQLException { + rs.setFetchDirection(ResultSet.FETCH_FORWARD); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + rs.setFetchDirection(ResultSet.FETCH_UNKNOWN); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void setFetchSize() throws SQLException { + rs.setFetchSize(0); + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getFetchSize() throws SQLException { + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, rs.getType()); + } + + @Test + public void getConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, rs.getConcurrency()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowUpdated() throws SQLException { + rs.rowUpdated(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowInserted() throws SQLException { + rs.rowInserted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowDeleted() throws SQLException { + rs.rowDeleted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNull() throws SQLException { + rs.updateNull("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBoolean() throws SQLException { + rs.updateBoolean(1, false); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateByte() throws SQLException { + rs.updateByte(1, new Byte("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateShort() throws SQLException { + rs.updateShort(1, new Short("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateInt() throws SQLException { + rs.updateInt(1, 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateLong() throws SQLException { + rs.updateLong(1, 1l); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateFloat() throws SQLException { + rs.updateFloat(1, 1f); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDouble() throws SQLException { + rs.updateDouble(1, 1.0); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBigDecimal() throws SQLException { + rs.updateBigDecimal(1, new BigDecimal(1)); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateString() throws SQLException { + rs.updateString(1, "abc"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBytes() throws SQLException { + rs.updateBytes(1, new byte[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDate() throws SQLException { + rs.updateDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTime() throws SQLException { + rs.updateTime(1, new Time(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTimestamp() throws SQLException { + rs.updateTimestamp(1, new Timestamp(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateAsciiStream() throws SQLException { + rs.updateAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBinaryStream() throws SQLException { + rs.updateBinaryStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateCharacterStream() throws SQLException { + rs.updateCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateObject() throws SQLException { + rs.updateObject(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void insertRow() throws SQLException { + rs.insertRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRow() throws SQLException { + rs.updateRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void deleteRow() throws SQLException { + rs.deleteRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void refreshRow() throws SQLException { + rs.refreshRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancelRowUpdates() throws SQLException { + rs.cancelRowUpdates(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToInsertRow() throws SQLException { + rs.moveToInsertRow(); + } + + @Test + public void getStatement() throws SQLException { + Statement stmt = rs.getStatement(); + Assert.assertNotNull(stmt); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToCurrentRow() throws SQLException { + rs.moveToCurrentRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRef() throws SQLException { + rs.getRef(1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBlob() throws SQLException { + rs.getBlob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getClob() throws SQLException { + rs.getClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getArray() throws SQLException { + rs.getArray("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getURL() throws SQLException { + rs.getURL("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRef() throws SQLException { + rs.updateRef("f1", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBlob() throws SQLException { + rs.updateBlob(1, (InputStream) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateClob() throws SQLException { + rs.updateClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateArray() throws SQLException { + rs.updateArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRowId() throws SQLException { + rs.getRowId("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRowId() throws SQLException { + rs.updateRowId(1, null); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, rs.getHoldability()); + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(rs.isClosed()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNString() throws SQLException { + rs.updateNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNClob() throws SQLException { + rs.updateNClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNClob() throws SQLException { + rs.getNClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getSQLXML() throws SQLException { + rs.getSQLXML("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateSQLXML() throws SQLException { + rs.updateSQLXML(1, null); + } + + @Test + public void getNString() throws SQLException { + String f10 = rs.getNString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getNString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNCharacterStream() throws SQLException { + rs.getNCharacterStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNCharacterStream() throws SQLException { + rs.updateNCharacterStream(1, null); + } + + @Test + public void unwrap() throws SQLException { + RestfulResultSet unwrap = rs.unwrap(RestfulResultSet.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(rs.isWrapperFor(RestfulResultSet.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..653e480413189667ec57d836f9b503f4c96a79cb --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java @@ -0,0 +1,418 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; +import java.util.UUID; + +public class RestfulStatementTest { + private static final String host = "127.0.0.1"; + // private static final String host = "master"; + private static Connection conn; + private static Statement stmt; + + @Test + public void executeQuery() { + try { + ResultSet rs = stmt.executeQuery("show databases"); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeUpdate() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + int affectRows = stmt.executeUpdate("create database " + dbName); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(1, affectRows); + affectRows = stmt.executeUpdate("drop database " + dbName); + Assert.assertEquals(0, affectRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void close() { + // test in AfterClass method + } + + + @Test + public void getMaxFieldSize() throws SQLException { + Assert.assertEquals(16 * 1024, stmt.getMaxFieldSize()); + } + + @Test(expected = SQLException.class) + public void setMaxFieldSize() throws SQLException { + + stmt.setMaxFieldSize(0); + stmt.setMaxFieldSize(-1); + } + + @Test + public void getMaxRows() throws SQLException { + Assert.assertEquals(0, stmt.getMaxRows()); + } + + @Test(expected = SQLException.class) + public void setMaxRows() throws SQLException { + stmt.setMaxRows(0); + stmt.setMaxRows(-1); + } + + @Test + public void setEscapeProcessing() throws SQLException { + stmt.setEscapeProcessing(true); + stmt.setEscapeProcessing(false); + } + + @Test + public void getQueryTimeout() throws SQLException { + Assert.assertEquals(0, stmt.getQueryTimeout()); + } + + @Test(expected = SQLException.class) + public void setQueryTimeout() throws SQLException { + stmt.setQueryTimeout(0); + stmt.setQueryTimeout(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancel() throws SQLException { + stmt.cancel(); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(stmt.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + stmt.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCursorName() throws SQLException { + stmt.setCursorName(""); + } + + @Test + public void execute() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database if not exists " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table if not exists " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getResultSet() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database if not exists " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table if not exists " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + ResultSet rs = stmt.getResultSet(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertEquals(3, meta.getColumnCount()); + int count = 0; + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + count++; + } + Assert.assertEquals(1, count); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getUpdateCount() { + // already test in execute method + } + + @Test + public void getMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults()); + } + + @Test(expected = SQLException.class) + public void setFetchDirection() throws SQLException { + stmt.setFetchDirection(ResultSet.FETCH_FORWARD); + stmt.setFetchDirection(ResultSet.FETCH_REVERSE); + stmt.setFetchDirection(ResultSet.FETCH_UNKNOWN); + stmt.setFetchDirection(-1); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, stmt.getFetchDirection()); + } + + @Test(expected = SQLException.class) + public void setFetchSize() throws SQLException { + stmt.setFetchSize(0); + stmt.setFetchSize(-1); + } + + @Test + public void getFetchSize() throws SQLException { + stmt.setFetchSize(0); + Assert.assertEquals(0, stmt.getFetchSize()); + stmt.setFetchSize(0); + } + + @Test + public void getResultSetConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, stmt.getResultSetConcurrency()); + } + + @Test + public void getResultSetType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, stmt.getResultSetType()); + } + + @Test + public void addBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void clearBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.clearBatch(); + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + stmt.clearBatch(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + int[] results = stmt.executeBatch(); + Assert.assertEquals(0, results[0]); + Assert.assertEquals(0, results[1]); + Assert.assertEquals(1, results[2]); + Assert.assertEquals(Statement.SUCCESS_NO_INFO, results[3]); + Assert.assertEquals(0, results[4]); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getConnection() { + try { + Connection connection = stmt.getConnection(); + Assert.assertNotNull(connection); + Assert.assertTrue(this.conn == connection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testGetMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults(Statement.CLOSE_CURRENT_RESULT)); + stmt.getMoreResults(Statement.KEEP_CURRENT_RESULT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getGeneratedKeys() throws SQLException { + stmt.getGeneratedKeys(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate() throws SQLException { + stmt.executeUpdate("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate1() throws SQLException { + stmt.executeUpdate("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate2() throws SQLException { + stmt.executeUpdate("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute() throws SQLException { + stmt.execute("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute1() throws SQLException { + stmt.execute("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute2() throws SQLException { + stmt.execute("", new String[]{}); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, stmt.getResultSetHoldability()); + } + + @Test + public void isClosed() { + try { + Assert.assertEquals(false, stmt.isClosed()); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void setPoolable() throws SQLException { + stmt.setPoolable(true); + stmt.setPoolable(false); + } + + @Test + public void isPoolable() throws SQLException { + Assert.assertEquals(false, stmt.isPoolable()); + } + + @Test + public void closeOnCompletion() throws SQLException { + stmt.closeOnCompletion(); + } + + @Test + public void isCloseOnCompletion() throws SQLException { + Assert.assertFalse(stmt.isCloseOnCompletion()); + } + + @Test + public void unwrap() throws SQLException { + RestfulStatement unwrap = stmt.unwrap(RestfulStatement.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(stmt.isWrapperFor(RestfulStatement.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata", properties); + stmt = conn.createStatement(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java new file mode 100644 index 0000000000000000000000000000000000000000..fe4d04775da642bcf258e3680b0f5ba822e69684 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -0,0 +1,349 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.utils.SQLExecutor; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import java.sql.*; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class SQLTest { + private static final String host = "127.0.0.1"; +// private static final String host = "master"; + private static Connection connection; + + @Test + public void testCase001() { + String sql = "create database if not exists restful_test"; + SQLExecutor.execute(connection, sql); + } + + @Test + public void testCase002() { + String sql = "use restful_test"; + SQLExecutor.execute(connection, sql); + } + + @Test + public void testCase003() { + String sql = "show databases"; + SQLExecutor.executeWithResult(connection, sql); + } + + @Test + public void testCase004() { + String sql = "show tables"; + SQLExecutor.executeWithResult(connection, sql); + } + + @Test + public void testCase005() { + String sql = "show stables"; + SQLExecutor.executeWithResult(connection, sql); + } + + @Test + public void testCase006() { + String sql = "show dnodes"; + SQLExecutor.executeWithResult(connection, sql); + } + + @Test + public void testCase007() { + String sql = "show vgroups"; + SQLExecutor.executeWithResult(connection, sql); + } + + @Test + public void testCase008() { + String sql = "drop table if exists restful_test.weather"; + SQLExecutor.execute(connection, sql); + } + + @Test + public void testCase009() { + String sql = "create table if not exists restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"; + SQLExecutor.execute(connection, sql); + } + + @Test + public void testCase010() { + String sql = "create table t1 using restful_test.weather tags('北京')"; + SQLExecutor.execute(connection, sql); + } + + @Test + public void testCase011() { + String sql = "insert into restful_test.t1 values(now, 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase012() { + String sql = "insert into restful_test.t1 values('2020-01-01 00:00:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase013() { + String sql = "insert into restful_test.t1 values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase014() { + String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:03:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase015() { + String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase016() { + String sql = "insert into t1 values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t2 values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase017() { + String sql = "Insert into t3 using weather tags('广东') values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t4 using weather tags('天津') values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase018() { + String sql = "select * from restful_test.t1"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase019() { + String sql = "select * from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase020() { + String sql = "select ts, temperature from restful_test.t1"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase021() { + String sql = "select ts, temperature from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase022() { + String sql = "select temperature, ts from restful_test.t1"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase023() { + String sql = "select temperature, ts from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase024() { + String sql = "import into restful_test.t5 using weather tags('石家庄') values('2020-01-01 00:01:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase025() { + String sql = "import into restful_test.t6 using weather tags('沈阳') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase026() { + String sql = "import into restful_test.t7 using weather tags('长沙') values('2020-01-01 00:01:00.000', 22.22) restful_test.t8 using weather tags('吉林') values('2020-01-01 00:01:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase027() { + String sql = "import into restful_test.t9 using weather tags('武汉') values('2020-01-01 00:01:00.000', 22.22) ,('2020-01-02 00:01:00.000', 22.22) restful_test.t10 using weather tags('哈尔滨') values('2020-01-01 00:01:00.000', 22.22),('2020-01-02 00:01:00.000', 22.22)"; + SQLExecutor.executeUpdate(connection, sql); + } + + @Test + public void testCase028() { + String sql = "select location, temperature, ts from restful_test.weather where temperature > 1"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase029() { + String sql = "select location, temperature, ts from restful_test.weather where temperature < 1"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase030() { + String sql = "select location, temperature, ts from restful_test.weather where ts > now"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase031() { + String sql = "select location, temperature, ts from restful_test.weather where ts < now"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase032() { + String sql = "select count(*) from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase033() { + String sql = "select first(*) from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase034() { + String sql = "select last(*) from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase035() { + String sql = "select last_row(*) from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase036() { + String sql = "select ts, ts as primary_key from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + + @Test + public void testCase037() { + String sql = "select database()"; + SQLExecutor.execute(connection, "use restful_test"); + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase038() { + String sql = "select client_version()"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase039() { + String sql = "select server_status()"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase040() { + String sql = "select server_status() as status"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase041() { + String sql = "select tbname, location from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase042() { + String sql = "select count(tbname) from restful_test.weather"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase043() { + String sql = "select * from restful_test.weather where ts < now - 1h"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase044() { + String sql = "select * from restful_test.weather where ts < now - 1h and location like '%'"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase045() { + String sql = "select * from restful_test.weather where ts < now - 1h order by ts"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase046() { + String sql = "select last(*) from restful_test.weather where ts < now - 1h group by tbname order by tbname"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase047() { + String sql = "select * from restful_test.weather limit 2"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase048() { + String sql = "select * from restful_test.weather limit 2 offset 5"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase049() { + String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts "; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase050() { + String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts and t1.location = t3.location"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase051() { + String sql = "select * from restful_test.t1 tt, restful_test.t3 yy where tt.ts = yy.ts"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase052() { + String sql = "select server_status()"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase053() { + String sql = "select avg(cpu_taosd), avg(cpu_system), max(cpu_cores), avg(mem_taosd), avg(mem_system), max(mem_total), avg(disk_used), max(disk_total), avg(band_speed), avg(io_read), avg(io_write), sum(req_http), sum(req_select), sum(req_insert) from log.dn1 where ts> now - 60m and ts<= now interval(1m) fill(value, 0)"; + SQLExecutor.executeQuery(connection, sql); + } + + @BeforeClass + public static void before() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + } + + @AfterClass + public static void after() throws SQLException { + connection.close(); + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java new file mode 100644 index 0000000000000000000000000000000000000000..bf034bf458bcb8eadaaacb5cf633f0905a8c1bd6 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java @@ -0,0 +1,74 @@ +package com.taosdata.jdbc.utils; + +import java.sql.*; + +public class SQLExecutor { + + // insert, import + public static void executeUpdate(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + int affectedRows = statement.executeUpdate(sql); + long end = System.currentTimeMillis(); + System.out.println("[ affected rows : " + affectedRows + " ] time cost: " + (end - start) + " ms, execute statement ====> " + sql); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // show databases, show tables, show stables + public static void executeWithResult(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + ResultSet resultSet = statement.getResultSet(); + printResult(resultSet); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // use database, create database, create table, drop table... + public static void execute(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + boolean execute = statement.execute(sql); + long end = System.currentTimeMillis(); + printSql(sql, execute, (end - start)); + } catch (SQLException e) { + System.out.println("ERROR execute SQL ===> " + sql); + e.printStackTrace(); + } + } + + // select + public static void executeQuery(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + long end = System.currentTimeMillis(); + printSql(sql, true, (end - start)); + printResult(resultSet); + } catch (SQLException e) { + System.out.println("ERROR execute SQL ===> " + sql); + e.printStackTrace(); + } + } + + private static void printSql(String sql, boolean succeed, long cost) { + System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private static void printResult(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder sb = new StringBuilder(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + sb.append(columnLabel + ": " + value + "\t"); + } + System.out.println(sb.toString()); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java index fb570e16c4e0ef8036ccd2c29cdc51e1938b2139..ccb0941da00e2755eaf8fde2553b8a8e6b33cd25 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -5,10 +5,6 @@ import org.junit.Test; public class SqlSyntaxValidatorTest { - @Test - public void validateSqlSyntax() { - } - @Test public void isSelectSQL() { Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather")); @@ -20,9 +16,6 @@ public class SqlSyntaxValidatorTest { @Test public void isUseSQL() { Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test")); } + } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimeStampUtil.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimeStampUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..1c6af7e3d2f1fd36fa620ab4a4dec77636d92db4 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimeStampUtil.java @@ -0,0 +1,67 @@ +package com.taosdata.jdbc.utils; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class TimeStampUtil { + + private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; + + public static long datetimeToLong(String dateTime) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + try { + return sdf.parse(dateTime).getTime(); + } catch (ParseException e) { + throw new IllegalArgumentException("invalid datetime string >>> " + dateTime); + } + } + + public static String longToDatetime(long time) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + return sdf.format(new Date(time)); + } + + public static class TimeTuple { + public Long start; + public Long end; + public Long timeGap; + + TimeTuple(long start, long end, long timeGap) { + this.start = start; + this.end = end; + this.timeGap = timeGap; + } + } + + public static TimeTuple range(long start, long timeGap, long size) { + long now = System.currentTimeMillis(); + if (timeGap < 1) + timeGap = 1; + if (start == 0) + start = now - size * timeGap; + + // 如果size小于1异常 + if (size < 1) + throw new IllegalArgumentException("size less than 1."); + // 如果timeGap为1，已经超长，需要前移start + if (start + size > now) { + start = now - size; + return new TimeTuple(start, now, 1); + } + long end = start + (long) (timeGap * size); + if (end > now) { + //压缩timeGap + end = now; + double gap = (end - start) / (size * 1.0f); + if (gap < 1.0f) { + timeGap = 1; + start = end - size; + } else { + timeGap = (long) gap; + end = start + (long) (timeGap * size); + } + } + return new TimeTuple(start, end, timeGap); + } +} diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 656741ea16fd607212711fb694e3e20e6007b2ac..43a08a800afe7aa3e445a943f8d7a7f95f1ed05f 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -3,14 +3,15 @@ * @module CTaosInterface */ -const ref = require('ref'); -const ffi = require('ffi'); -const ArrayType = require('ref-array'); -const Struct = require('ref-struct'); +const ref = require('ref-napi'); +const os = require('os'); +const ffi = require('ffi-napi'); +const ArrayType = require('ref-array-napi'); +const Struct = require('ref-struct-napi'); const FieldTypes = require('./constants'); const errors = require ('./error'); const TaosObjects = require('./taosobjects'); -const { NULL_POINTER } = require('ref'); +const { NULL_POINTER } = require('ref-napi'); module.exports = CTaosInterface; @@ -144,18 +145,9 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; - let currOffset = 0; - // every 4 bytes, a character is encoded; - while (currOffset < data.length) { - let dataEntry = data.slice(currOffset, currOffset + nbytes); //one entry in a row under a column; - if (dataEntry.readInt64LE(0) == FieldTypes.C_NCHAR_NULL) { - res.push(null); - } - else { - res.push(dataEntry.toString("utf16le").replace(/\u0000/g, "")); - } - currOffset += nbytes; - } + let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; + //TODO: should use the correct character encoding + res.push(dataEntry.toString("utf-8")); return res; } @@ -197,7 +189,13 @@ function CTaosInterface (config = null, pass = false) { ref.types.void_ptr2 = ref.refType(ref.types.void_ptr); /*Declare a bunch of functions first*/ /* Note, pointers to TAOS_RES, TAOS, are ref.types.void_ptr. The connection._conn buffer is supplied for pointers to TAOS * */ - this.libtaos = ffi.Library('libtaos', { + + if ('win32' == os.platform()) { + taoslibname = 'taos'; + } else { + taoslibname = 'libtaos'; + } + this.libtaos = ffi.Library(taoslibname, { 'taos_options': [ ref.types.int, [ ref.types.int , ref.types.void_ptr ] ], 'taos_init': [ ref.types.void, [ ] ], //TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) @@ -349,11 +347,13 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let num_of_rows = this.libtaos.taos_fetch_block(result, pblock) - if (num_of_rows == 0) { + //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data + let pblock = this.libtaos.taos_fetch_row(result); + let num_of_rows = 1; + if (ref.isNull(pblock) == true) { return {block:null, num_of_rows:0}; } + var fieldL = this.libtaos.taos_fetch_lengths(result); let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); @@ -361,26 +361,28 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { var fieldlens = []; if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + let plen = ref.reinterpret(fieldL, 4, i*4); let len = plen.readInt32LE(0); - fieldlens.push(len); + fieldlens.push(len); } } let blocks = new Array(fields.length); blocks.fill(null); - num_of_rows = Math.abs(num_of_rows); + //num_of_rows = Math.abs(num_of_rows); let offset = 0; - pblock = pblock.deref(); for (let i = 0; i < fields.length; i++) { pdata = ref.reinterpret(pblock,8,i*8); - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + } } return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} } @@ -446,14 +448,18 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, } if (numOfRows2 > 0){ for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row,8,i*8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index 0c9214fe32656fdfb7edb6cfc0d0c3587d4f7bb6..e18e6c2500019684dea85fe05bc7306e0a372478 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -1,4 +1,4 @@ -const ref = require('ref'); +const ref = require('ref-napi'); require('./globalfunc.js') const CTaosInterface = require('./cinterface') const errors = require ('./error') diff --git a/src/connector/nodejs/nodetaos/taosresult.js b/src/connector/nodejs/nodetaos/taosresult.js index fd82f4e236508eff46c54f6f156b592a43319c20..4138ebbec6e1b792691d17a25b7c18d35b6a922a 100644 --- a/src/connector/nodejs/nodetaos/taosresult.js +++ b/src/connector/nodejs/nodetaos/taosresult.js @@ -25,6 +25,7 @@ function TaosResult(data, fields) { * @function pretty * @since 1.0.6 */ + TaosResult.prototype.pretty = function pretty() { let fieldsStr = ""; let sizing = []; @@ -46,8 +47,7 @@ TaosResult.prototype.pretty = function pretty() { row.data.forEach((entry, i) => { if (this.fields[i]._field.type == 9) { entry = entry.toTaosString(); - } - else { + } else { entry = entry == null ? 'null' : entry.toString(); } rowStr += entry diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json index d13fe6959c07955f5b15c7281f571ab932a996c3..9ca174ccd12f782b047789e44a6abce231b15ad0 100644 --- a/src/connector/nodejs/package-lock.json +++ b/src/connector/nodejs/package-lock.json @@ -1,44 +1,9 @@ { "name": "td2.0-connector", - "version": "2.0.1", + "version": "2.0.6", "lockfileVersion": 1, "requires": true, "dependencies": { - "abbrev": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" - }, - "ajv": { - "version": "6.10.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", - "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==", - "requires": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" - }, - "are-we-there-yet": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", - "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, "array-index": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz", @@ -46,104 +11,23 @@ "requires": { "debug": "^2.2.0", "es6-symbol": "^3.0.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } }, - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "requires": { - "safer-buffer": "~2.1.0" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" - }, - "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "requires": { - "tweetnacl": "^0.14.3" - } - }, - "bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "requires": { - "file-uri-to-path": "1.0.0" - } - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "chownr": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz", - "integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==" - }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" - }, - "combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "d": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", @@ -153,54 +37,22 @@ "type": "^1.0.1" } }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "^1.0.0" - } - }, "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" - }, - "ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "requires": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" + "ms": "2.1.2" } }, - "env-paths": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-1.0.0.tgz", - "integrity": "sha1-QWgTO0K7BcOKNbGuQ5fIKYqzaeA=" - }, "es5-ext": { - "version": "0.10.50", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.50.tgz", - "integrity": "sha512-KMzZTPBkeQV/JcSQhI5/z6d9VWJ3EnQ194USTUwIYZ2ZbpN8+SGXQKt1h68EX44+qt+Fzr8DO17vnxrw7c3agw==", + "version": "0.10.53", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", + "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", "requires": { "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.1", - "next-tick": "^1.0.0" + "es6-symbol": "~3.1.3", + "next-tick": "~1.0.0" } }, "es6-iterator": { @@ -214,631 +66,220 @@ } }, "es6-symbol": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz", - "integrity": "sha1-vwDvT9q2uhtG7Le2KbTH7VcVzHc=", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", + "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", "requires": { - "d": "1", - "es5-ext": "~0.10.14" + "d": "^1.0.1", + "ext": "^1.1.2" } }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, - "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" - }, - "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "ffi": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/ffi/-/ffi-2.3.0.tgz", - "integrity": "sha512-vkPA9Hf9CVuQ5HeMZykYvrZF2QNJ/iKGLkyDkisBnoOOFeFXZQhUPxBARPBIZMJVulvBI2R+jgofW03gyPpJcQ==", + "ext": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", + "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", "requires": { - "bindings": "~1.2.0", - "debug": "2", - "nan": "2", - "ref": "1", - "ref-struct": "1" + "type": "^2.0.0" }, "dependencies": { - "bindings": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", - "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE=" + "type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", + "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" } } }, - "file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "fs-minipass": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.6.tgz", - "integrity": "sha512-crhvyXcMejjv3Z5d2Fa9sf5xLYVCF5O1c71QxbVnbLsmYMBEvDAftewesN/HhY03YRoA7zOMxjNGrF5svGaaeQ==", - "requires": { - "minipass": "^2.2.1" - } - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "graceful-fs": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", - "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==" - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "ffi-napi": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz", + "integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==", "requires": { - "ajv": "^6.5.5", - "har-schema": "^2.0.0" + "debug": "^4.1.1", + "get-uv-event-loop-napi-h": "^1.0.5", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1", + "ref-napi": "^2.0.1", + "ref-struct-di": "^1.1.0" + }, + "dependencies": { + "ref-napi": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz", + "integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==", + "requires": { + "debug": "^4.1.1", + "get-symbol-from-current-process-h": "^1.0.2", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" + } + } } }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" - }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "requires": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - } + "get-symbol-from-current-process-h": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", + "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" }, - "inflight": { + "get-uv-event-loop-napi-h": { "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", + "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", "requires": { - "number-is-nan": "^1.0.0" - } - }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "mime-db": { - "version": "1.40.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", - "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" - }, - "mime-types": { - "version": "2.1.24", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", - "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", - "requires": { - "mime-db": "1.40.0" - } - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - }, - "minipass": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", - "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", - "requires": { - "safe-buffer": "^5.1.2", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", - "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", - "requires": { - "minipass": "^2.2.1" - } - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" + "get-symbol-from-current-process-h": "^1.0.1" } }, "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "nan": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", - "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "next-tick": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" }, - "node-gyp": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-5.0.2.tgz", - "integrity": "sha512-sNcb5O7eJ9XiNAhWZ/UE2bWsBJn3Jb7rayMqMP4wjenlr1DwzZxUmbtmIrl04EU0p5fN2rU9WIDV+u0EbsI8oQ==", - "requires": { - "env-paths": "^1.0.0", - "glob": "^7.0.3", - "graceful-fs": "^4.1.2", - "mkdirp": "^0.5.0", - "nopt": "2 || 3", - "npmlog": "0 || 1 || 2 || 3 || 4", - "request": "^2.87.0", - "rimraf": "2", - "semver": "~5.3.0", - "tar": "^4.4.8", - "which": "1" - } - }, - "nopt": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", - "requires": { - "abbrev": "1" - } - }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" - }, - "oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" - }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "psl": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.2.0.tgz", - "integrity": "sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA==" - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "node-addon-api": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", + "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "node-gyp-build": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", + "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==" }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "ref": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ref/-/ref-1.3.5.tgz", - "integrity": "sha512-2cBCniTtxcGUjDpvFfVpw323a83/0RLSGJJY5l5lcomZWhYpU2cuLdsvYqMixvsdLJ9+sTdzEkju8J8ZHDM2nA==", - "requires": { - "bindings": "1", - "debug": "2", - "nan": "2" - } - }, - "ref-array": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/ref-array/-/ref-array-1.2.0.tgz", - "integrity": "sha1-u6hwFS1O4KvtFCGwYjkvCwGEKQw=", + "ref-array-napi": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz", + "integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==", "requires": { "array-index": "1", "debug": "2", - "ref": "1" - } - }, - "ref-struct": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ref-struct/-/ref-struct-1.1.0.tgz", - "integrity": "sha1-XV7mWtQc78Olxf60BYcmHkee3BM=", - "requires": { - "debug": "2", - "ref": "1" - } - }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, - "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "requires": { - "glob": "^7.1.3" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "semver": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", - "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=" - }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, - "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" - }, - "sshpk": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", - "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "requires": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" + "ref-napi": "^1.4.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "ref-napi": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", + "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", + "requires": { + "debug": "^3.1.0", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + } } }, - "strip-ansi": { + "ref-napi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "tar": { - "version": "4.4.10", - "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz", - "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz", + "integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==", "requires": { - "chownr": "^1.1.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.3.5", - "minizlib": "^1.2.1", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.2", - "yallist": "^3.0.3" + "debug": "^4.1.1", + "get-symbol-from-current-process-h": "^1.0.2", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" } }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "ref-struct-di": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", + "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" + "debug": "^3.1.0" }, "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } } } }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "ref-struct-napi": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz", + "integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==", "requires": { - "safe-buffer": "^5.0.1" + "debug": "2", + "ref-napi": "^1.4.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "ref-napi": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", + "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", + "requires": { + "debug": "^3.1.0", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + } } }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, "type": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/type/-/type-1.0.1.tgz", - "integrity": "sha512-MAM5dBMJCJNKs9E7JXo4CXRAansRfG0nlJxW7Wf6GZzSOvH31zClSaHdIMWLehe/EGMBkqeC55rrkaOr5Oo7Nw==" - }, - "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "requires": { - "punycode": "^2.1.0" - } - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "requires": { - "isexe": "^2.0.0" - } - }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "requires": { - "string-width": "^1.0.2 || 2" - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" } } } diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index 3f0600a09ca73c3fadbed1b7e65c51b30004edf5..b39ce2c17d8e693d0e981885aed55ba1b82313b6 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,31 +1,34 @@ { "name": "td2.0-connector", - "version": "2.0.1", + "version": "2.0.6", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", + "directories": { + "test": "test" + }, "scripts": { "test": "node test/test.js" }, + "repository": { + "type": "git", + "url": "git+https://github.com/taosdata/tdengine.git" + }, "keywords": [ "TDengine", "TAOS Data", "Time Series Database", "Connector" ], - "author": "StoneT2000", - "license": "AGPL-3.0", - "dependencies": { - "ffi": "^2.3.0", - "node-gyp": "^5.0.2", - "ref": "^1.3.5", - "ref-array": "^1.2.0" - }, - "homepage": "https://github.com/taosdata/tdengine", + "author": "TaosData Inc.", + "license": "AGPL-3.0-or-later", "bugs": { "url": "https://github.com/taosdata/tdengine/issues" }, - "repository": { - "type": "git", - "url": "https://github.com/taosdata/tdengine.git" + "homepage": "https://github.com/taosdata/tdengine#readme", + "dependencies": { + "ffi-napi": "^3.1.0", + "ref-array-napi": "^1.2.1", + "ref-napi": "^1.5.2", + "ref-struct-napi": "^1.1.1" } } diff --git a/src/connector/nodejs/test/test.js b/src/connector/nodejs/test/test.js index 73dac8b26ce07ff159ce3de535776e2495d1ebde..bf4bb2c54188d3eb0f9c7fb5306912effc7b0760 100644 --- a/src/connector/nodejs/test/test.js +++ b/src/connector/nodejs/test/test.js @@ -48,6 +48,7 @@ for (let i = 0; i < 10000; i++) { // Select console.log('select * from td_connector_test.all_types limit 3 offset 100;'); c1.execute('select * from td_connector_test.all_types limit 2 offset 100;'); + var d = c1.fetchall(); console.log(c1.fields); console.log(d); @@ -77,13 +78,33 @@ c1.query('select stddev(_double), stddev(_bigint), stddev(_float) from all_types }) // Binding arguments, and then using promise -var q = c1.query('select * from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) +var q = c1.query('select _nchar from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) console.log(q.query); q.execute().then(function(r) { r.pretty(); }); +// test query null value +c1.execute("create table if not exists td_connector_test.weather(ts timestamp, temperature float, humidity int) tags(location nchar(64))"); +c1.execute("insert into t1 using weather tags('北京') values(now, 11.11, 11)"); +c1.execute("insert into t1(ts, temperature) values(now, 22.22)"); +c1.execute("insert into t1(ts, humidity) values(now, 33)"); +c1.query('select * from test.t1', true).then(function (result) { + result.pretty(); +}); + +var q = c1.query('select * from td_connector_test.weather'); +console.log(q.query); +q.execute().then(function(r) { + r.pretty(); +}); + +function sleep(sleepTime) { + for(var start = +new Date; +new Date - start <= sleepTime; ) { } +} + +sleep(10000); // Raw Async Testing (Callbacks, not promises) function cb2(param, result, rowCount, rd) { @@ -129,16 +150,21 @@ setTimeout(function(){ c1.fetchall_a(thisRes, cb4, param); },100); + // Async through promises var aq = c1.query('select count(*) from td_connector_test.all_types;',false); aq.execute_a().then(function(data) { data.pretty(); }); -c1.query('describe td_connector_test.stabletest;').execute_a().then(r=> r.pretty()); + +c1.query('describe td_connector_test.stabletest').execute_a().then(function(r){ + r.pretty() +}); + setTimeout(function(){ c1.query('drop database td_connector_test;'); },200); + setTimeout(function(){ conn.close(); },2000); - diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index 92a931b504fd76d12c1a4094a04fed4f859c0a3e..4a829f36c4bf0d6e680ed923573509cc1fad39db 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.6", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py index 62e0536b6fca63a0aca0b6be3673104c27c631c6..973263573808232e4e71dc0158585624a8e7d2ab 100644 --- a/src/connector/python/linux/python2/taos/__init__.py +++ b/src/connector/python/linux/python2/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 2b1b29ee31af56499d626fc56d4950e2e25f9978..555cc3435bcbea302b34cbde09772ac5f6fe32b2 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,168 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) - return res + return res + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.CDLL('libtaos.so') @@ -216,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -227,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -236,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -255,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -268,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -293,12 +443,13 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -308,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -360,38 +511,53 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) + @staticmethod def freeResult(result): CTaosInterface.libtaos.taos_free_result(result) diff --git a/src/connector/python/linux/python2/taos/connection.py b/src/connector/python/linux/python2/taos/connection.py index 552250f1164ced467cd29b5084524985aca8848b..f6c395342c9c39a24bda6022f0ed36cb7bfe045b 100644 --- a/src/connector/python/linux/python2/taos/connection.py +++ b/src/connector/python/linux/python2/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -29,7 +31,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -43,7 +45,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -55,7 +62,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -80,7 +88,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/linux/python2/taos/constants.py b/src/connector/python/linux/python2/taos/constants.py index feb7050a40b67f88a6d7ca859764fbbc6b36af1c..93466f5184a6bf37c2e1c915a00aa5c5e91d1801 100644 --- a/src/connector/python/linux/python2/taos/constants.py +++ b/src/connector/python/linux/python2/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index 82a01be67118813f9c83c3c3870d91535e13c526..8f9aab82da64d24645311d1263f9abb006c737eb 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading class TDengineCursor(object): @@ -36,7 +35,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection @@ -130,8 +128,8 @@ class TDengineCursor(object): if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: self._affected_rows += CTaosInterface.affectedRows( - self._result ) - return CTaosInterface.affectedRows(self._result ) + self._result) + return CTaosInterface.affectedRows(self._result) else: self._fields = CTaosInterface.useResult( self._result) @@ -150,6 +148,7 @@ class TDengineCursor(object): """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. """ pass + def fetchmany(self): pass @@ -160,11 +159,26 @@ class TDengineCursor(object): if (dataType.upper() == "TINYINT"): if (self._description[col][1] == FieldType.C_TINYINT): return True + if (dataType.upper() == "TINYINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + return True + if (dataType.upper() == "SMALLINT"): + if (self._description[col][1] == FieldType.C_SMALLINT): + return True + if (dataType.upper() == "SMALLINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + return True if (dataType.upper() == "INT"): if (self._description[col][1] == FieldType.C_INT): return True + if (dataType.upper() == "INT UNSIGNED"): + if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + return True if (dataType.upper() == "BIGINT"): - if (self._description[col][1] == FieldType.C_INT): + if (self._description[col][1] == FieldType.C_BIGINT): + return True + if (dataType.upper() == "BIGINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): return True if (dataType.upper() == "FLOAT"): if (self._description[col][1] == FieldType.C_FLOAT): @@ -184,7 +198,7 @@ class TDengineCursor(object): return False - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -193,33 +207,41 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def fetchall_block(self): + + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def nextset(self): """ """ diff --git a/src/connector/python/linux/python2/taos/dbapi.py b/src/connector/python/linux/python2/taos/dbapi.py index f1c22bdb512224ac712b78b15ec00207587e65c5..594681ada953abf388e503c23199043cf686e1a3 100644 --- a/src/connector/python/linux/python2/taos/dbapi.py +++ b/src/connector/python/linux/python2/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/linux/python2/taos/error.py b/src/connector/python/linux/python2/taos/error.py index 24508a72ed78bb6231187bb6de34d57182e31b22..c584badce8320cd35dc81e8f6b613c56163b1a29 100644 --- a/src/connector/python/linux/python2/taos/error.py +++ b/src/connector/python/linux/python2/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/linux/python2/taos/subscription.py b/src/connector/python/linux/python2/taos/subscription.py index 2d01395532820c3bd0e068ef7eb3d425eaaa6d78..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/linux/python2/taos/subscription.py +++ b/src/connector/python/linux/python2/taos/subscription.py @@ -1,52 +1,57 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: return False - + CTaosInterface.unsubscribe(self._sub, keepProgress) return True if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index 655a12ad135e4eae575061e6069724a12a2a6996..a865f5df856d1b416d7f78da8b1f857a967f5e61 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.5", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py index 8cf095ea68dda55a071403734d39e9198a71d8a1..973263573808232e4e71dc0158585624a8e7d2ab 100644 --- a/src/connector/python/linux/python3/taos/__init__.py +++ b/src/connector/python/linux/python3/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index fdebe349fe9597260f231adb52022a2ecc9e1063..555cc3435bcbea302b34cbde09772ac5f6fe32b2 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,168 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) - return res + return res + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.CDLL('libtaos.so') @@ -216,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -227,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -236,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -255,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -268,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -293,7 +443,8 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: @@ -308,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -360,35 +511,49 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) diff --git a/src/connector/python/linux/python3/taos/connection.py b/src/connector/python/linux/python3/taos/connection.py index 552250f1164ced467cd29b5084524985aca8848b..f6c395342c9c39a24bda6022f0ed36cb7bfe045b 100644 --- a/src/connector/python/linux/python3/taos/connection.py +++ b/src/connector/python/linux/python3/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -29,7 +31,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -43,7 +45,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -55,7 +62,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -80,7 +88,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/linux/python3/taos/constants.py b/src/connector/python/linux/python3/taos/constants.py index feb7050a40b67f88a6d7ca859764fbbc6b36af1c..93466f5184a6bf37c2e1c915a00aa5c5e91d1801 100644 --- a/src/connector/python/linux/python3/taos/constants.py +++ b/src/connector/python/linux/python3/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index 0ce20f0edaabe5cf0d2bb6efa3e031f51779f03d..2e7c362d547973bc3e78f2eb57a33b7fa2d0635e 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -5,6 +5,7 @@ import threading # querySeqNum = 0 + class TDengineCursor(object): """Database cursor which is used to manage the context of a fetch operation. @@ -107,8 +108,8 @@ class TDengineCursor(object): # if threading.get_ident() != self._threadId: # info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) # raise OperationalError(info) - # print(info) - # return None + # print(info) + # return None if not operation: return None @@ -137,8 +138,8 @@ class TDengineCursor(object): if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: self._affected_rows += CTaosInterface.affectedRows( - self._result ) - return CTaosInterface.affectedRows(self._result ) + self._result) + return CTaosInterface.affectedRows(self._result) else: self._fields = CTaosInterface.useResult( self._result) @@ -168,11 +169,26 @@ class TDengineCursor(object): if (dataType.upper() == "TINYINT"): if (self._description[col][1] == FieldType.C_TINYINT): return True + if (dataType.upper() == "TINYINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + return True + if (dataType.upper() == "SMALLINT"): + if (self._description[col][1] == FieldType.C_SMALLINT): + return True + if (dataType.upper() == "SMALLINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + return True if (dataType.upper() == "INT"): if (self._description[col][1] == FieldType.C_INT): return True + if (dataType.upper() == "INT UNSIGNED"): + if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + return True if (dataType.upper() == "BIGINT"): - if (self._description[col][1] == FieldType.C_INT): + if (self._description[col][1] == FieldType.C_BIGINT): + return True + if (dataType.upper() == "BIGINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): return True if (dataType.upper() == "FLOAT"): if (self._description[col][1] == FieldType.C_FLOAT): @@ -192,7 +208,7 @@ class TDengineCursor(object): return False - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -201,10 +217,13 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -212,22 +231,27 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def fetchall_block(self): + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) - if num_of_fields == 0: break + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def nextset(self): """ """ @@ -259,8 +283,8 @@ class TDengineCursor(object): # if threading.get_ident() != self._threadId: # info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) # raise OperationalError(info) - # print(info) - # return None + # print(info) + # return None self._description = [] for ele in self._fields: @@ -268,4 +292,3 @@ class TDengineCursor(object): (ele['name'], ele['type'], None, None, None, None, False)) return self._result - diff --git a/src/connector/python/linux/python3/taos/dbapi.py b/src/connector/python/linux/python3/taos/dbapi.py index f1c22bdb512224ac712b78b15ec00207587e65c5..594681ada953abf388e503c23199043cf686e1a3 100644 --- a/src/connector/python/linux/python3/taos/dbapi.py +++ b/src/connector/python/linux/python3/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/linux/python3/taos/error.py b/src/connector/python/linux/python3/taos/error.py index 24508a72ed78bb6231187bb6de34d57182e31b22..c584badce8320cd35dc81e8f6b613c56163b1a29 100644 --- a/src/connector/python/linux/python3/taos/error.py +++ b/src/connector/python/linux/python3/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/linux/python3/taos/subscription.py b/src/connector/python/linux/python3/taos/subscription.py index d3cf10d5ada578687689b94454378dd543368e3e..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/linux/python3/taos/subscription.py +++ b/src/connector/python/linux/python3/taos/subscription.py @@ -1,32 +1,33 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: @@ -38,15 +39,19 @@ class TDengineSubscription(object): if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/connector/python/osx/python3/LICENSE b/src/connector/python/osx/python3/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..79a9d730868bfe5d3fa01d679a4abfe9ee7811f0 --- /dev/null +++ b/src/connector/python/osx/python3/LICENSE @@ -0,0 +1,12 @@ + Copyright (c) 2019 TAOS Data, Inc. + +This program is free software: you can use, redistribute, and/or modify +it under the terms of the GNU Affero General Public License, version 3 +or later ("AGPL"), as published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . diff --git a/src/connector/python/osx/python3/README.md b/src/connector/python/osx/python3/README.md new file mode 100644 index 0000000000000000000000000000000000000000..70db6bba13a8b52b9f707400b80d1302542dbc34 --- /dev/null +++ b/src/connector/python/osx/python3/README.md @@ -0,0 +1 @@ +# TDengine python client interface \ No newline at end of file diff --git a/src/connector/python/osx/python3/setup.py b/src/connector/python/osx/python3/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..a6b97f753c7ee3bc303be0db2217e87e889ef4df --- /dev/null +++ b/src/connector/python/osx/python3/setup.py @@ -0,0 +1,20 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="taos", + version="2.0.5", + author="Taosdata Inc.", + author_email="support@taosdata.com", + description="TDengine python client package", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/pypa/sampleproject", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: MacOS X", + ], +) diff --git a/src/connector/python/osx/python3/taos/__init__.py b/src/connector/python/osx/python3/taos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..973263573808232e4e71dc0158585624a8e7d2ab --- /dev/null +++ b/src/connector/python/osx/python3/taos/__init__.py @@ -0,0 +1,24 @@ + +from .connection import TDengineConnection +from .cursor import TDengineCursor + +# Globals +threadsafety = 0 +paramstyle = 'pyformat' + +__all__ = ['connection', 'cursor'] + + +def connect(*args, **kwargs): + """ Function to return a TDengine connector object + + Current supporting keyword parameters: + @dsn: Data source name as string + @user: Username as string(optional) + @password: Password as string(optional) + @host: Hostname(optional) + @database: Database name(optional) + + @rtype: TDengineConnector + """ + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/osx/python3/taos/cinterface.py b/src/connector/python/osx/python3/taos/cinterface.py new file mode 100644 index 0000000000000000000000000000000000000000..6f56cf0c5e09c14fdc9d1296c80e434ab672ef44 --- /dev/null +++ b/src/connector/python/osx/python3/taos/cinterface.py @@ -0,0 +1,642 @@ +import ctypes +from .constants import FieldType +from .error import * +import math +import datetime + + +def _convert_millisecond_to_datetime(milli): + return datetime.datetime.fromtimestamp(milli / 1000.0) + + +def _convert_microsecond_to_datetime(micro): + return datetime.datetime.fromtimestamp(micro / 1000000.0) + + +def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C bool row to python row + """ + _timestamp_converter = _convert_millisecond_to_datetime + if micro: + _timestamp_converter = _convert_microsecond_to_datetime + + if num_of_rows > 0: + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + else: + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + + +def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C bool row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + + +def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + +def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + + +def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + + +def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] + + +def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C float row to python row + """ + if num_of_rows > 0: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + else: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + + +def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C double row to python row + """ + if num_of_rows > 0: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + else: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + + +def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + assert(nbytes is not None) + if num_of_rows > 0: + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + else: + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + + +def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res = [] + for i in range(abs(num_of_rows)): + try: + if num_of_rows >= 0: + tmpstr = ctypes.c_char_p(data) + res.append(tmpstr.value.decode()) + else: + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) + except ValueError: + res.append(None) + + return res + + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + assert(nbytes is not None) + res = [] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) + except ValueError: + res.append(None) + return res + + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res = [] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) + except ValueError: + res.append(None) + return res + + +_CONVERT_FUNC = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python +} + +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python +} + +# Corresponding TAOS_FIELD structure in C + + +class TaosField(ctypes.Structure): + _fields_ = [('name', ctypes.c_char * 65), + ('type', ctypes.c_char), + ('bytes', ctypes.c_short)] + +# C interface class + + +class CTaosInterface(object): + + libtaos = ctypes.CDLL('libtaos.dylib') + + libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) + libtaos.taos_init.restype = None + libtaos.taos_connect.restype = ctypes.c_void_p + #libtaos.taos_use_result.restype = ctypes.c_void_p + libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) + libtaos.taos_errstr.restype = ctypes.c_char_p + libtaos.taos_subscribe.restype = ctypes.c_void_p + libtaos.taos_consume.restype = ctypes.c_void_p + libtaos.taos_fetch_lengths.restype = ctypes.c_void_p + libtaos.taos_free_result.restype = None + libtaos.taos_errno.restype = ctypes.c_int + libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) + + def __init__(self, config=None): + ''' + Function to initialize the class + @host : str, hostname to connect + @user : str, username to connect to server + @password : str, password to connect to server + @db : str, default db to use when log in + @config : str, config directory + + @rtype : None + ''' + if config is None: + self._config = ctypes.c_char_p(None) + else: + try: + self._config = ctypes.c_char_p(config.encode('utf-8')) + except AttributeError: + raise AttributeError("config is expected as a str") + + if config is not None: + CTaosInterface.libtaos.taos_options(3, self._config) + + CTaosInterface.libtaos.taos_init() + + @property + def config(self): + """ Get current config + """ + return self._config + + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): + ''' + Function to connect to server + + @rtype: c_void_p, TDengine handle + ''' + # host + try: + _host = ctypes.c_char_p(host.encode( + "utf-8")) if host is not None else ctypes.c_char_p(None) + except AttributeError: + raise AttributeError("host is expected as a str") + + # user + try: + _user = ctypes.c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + + # password + try: + _password = ctypes.c_char_p(password.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = ctypes.c_char_p( + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) + except AttributeError: + raise AttributeError("db is expected as a str") + + # port + try: + _port = ctypes.c_int(port) + except TypeError: + raise TypeError("port is expected as an int") + + connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( + _host, _user, _password, _db, _port)) + + if connection.value is None: + print('connect to TDengine failed') + raise ConnectionError("connect to TDengine failed") + # sys.exit(1) + # else: + # print('connect to TDengine success') + + return connection + + @staticmethod + def close(connection): + '''Close the TDengine handle + ''' + CTaosInterface.libtaos.taos_close(connection) + #print('connection is closed') + + @staticmethod + def query(connection, sql): + '''Run SQL + + @sql: str, sql string to run + + @rtype: 0 on success and -1 on failure + ''' + try: + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) + except AttributeError: + raise AttributeError("sql is expected as a string") + # finally: + # CTaosInterface.libtaos.close(connection) + + @staticmethod + def affectedRows(result): + """The affected rows after runing query + """ + return CTaosInterface.libtaos.taos_affected_rows(result) + + @staticmethod + def subscribe(connection, restart, topic, sql, interval): + """Create a subscription + @restart boolean, + @sql string, sql statement for data query, must be a 'select' statement. + @topic string, name of this subscription + """ + return ctypes.c_void_p(CTaosInterface.libtaos.taos_subscribe( + connection, + 1 if restart else 0, + ctypes.c_char_p(topic.encode('utf-8')), + ctypes.c_char_p(sql.encode('utf-8')), + None, + None, + interval)) + + @staticmethod + def consume(sub): + """Consume data of a subscription + """ + result = ctypes.c_void_p(CTaosInterface.libtaos.taos_consume(sub)) + fields = [] + pfields = CTaosInterface.fetchFields(result) + for i in range(CTaosInterface.libtaos.taos_num_fields(result)): + fields.append({'name': pfields[i].name.decode('utf-8'), + 'bytes': pfields[i].bytes, + 'type': ord(pfields[i].type)}) + return result, fields + + @staticmethod + def unsubscribe(sub, keepProgress): + """Cancel a subscription + """ + CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) + + @staticmethod + def useResult(result): + '''Use result after calling self.query + ''' + fields = [] + pfields = CTaosInterface.fetchFields(result) + for i in range(CTaosInterface.fieldsCount(result)): + fields.append({'name': pfields[i].name.decode('utf-8'), + 'bytes': pfields[i].bytes, + 'type': ord(pfields[i].type)}) + + return fields + + @staticmethod + def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + + @staticmethod + def fetchRow(result, fields): + pblock = ctypes.c_void_p(0) + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: + num_of_rows = 1 + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC: + raise DatabaseError( + "Invalid data type returned from database") + if data is None: + blocks[i] = [None] + else: + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) + else: + return None, 0 + return blocks, abs(num_of_rows) + + @staticmethod + def freeResult(result): + CTaosInterface.libtaos.taos_free_result(result) + result.value = None + + @staticmethod + def fieldsCount(result): + return CTaosInterface.libtaos.taos_field_count(result) + + @staticmethod + def fetchFields(result): + return CTaosInterface.libtaos.taos_fetch_fields(result) + + # @staticmethod + # def fetchRow(result, fields): + # l = [] + # row = CTaosInterface.libtaos.taos_fetch_row(result) + # if not row: + # return None + + # for i in range(len(fields)): + # l.append(CTaosInterface.getDataValue( + # row[i], fields[i]['type'], fields[i]['bytes'])) + + # return tuple(l) + + # @staticmethod + # def getDataValue(data, dtype, byte): + # ''' + # ''' + # if not data: + # return None + + # if (dtype == CTaosInterface.TSDB_DATA_TYPE_BOOL): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TINYINT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_SMALLINT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): + # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): + # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') + + @staticmethod + def errno(result): + """Return the error number. + """ + return CTaosInterface.libtaos.taos_errno(result) + + @staticmethod + def errStr(result): + """Return the error styring + """ + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') + + +if __name__ == '__main__': + cinter = CTaosInterface() + conn = cinter.connect() + result = cinter.query(conn, 'show databases') + + print('Query Affected rows: {}'.format(cinter.affectedRows(result))) + + fields = CTaosInterface.useResult(result) + + data, num_of_rows = CTaosInterface.fetchBlock(result, fields) + + print(data) + + cinter.freeResult(result) + cinter.close(conn) diff --git a/src/connector/python/osx/python3/taos/connection.py b/src/connector/python/osx/python3/taos/connection.py new file mode 100644 index 0000000000000000000000000000000000000000..f6c395342c9c39a24bda6022f0ed36cb7bfe045b --- /dev/null +++ b/src/connector/python/osx/python3/taos/connection.py @@ -0,0 +1,95 @@ +from .cursor import TDengineCursor +from .subscription import TDengineSubscription +from .cinterface import CTaosInterface + + +class TDengineConnection(object): + """ TDengine connection object + """ + + def __init__(self, *args, **kwargs): + self._conn = None + self._host = None + self._user = "root" + self._password = "taosdata" + self._database = None + self._port = 0 + self._config = None + self._chandle = None + + self.config(**kwargs) + + def config(self, **kwargs): + # host + if 'host' in kwargs: + self._host = kwargs['host'] + + # user + if 'user' in kwargs: + self._user = kwargs['user'] + + # password + if 'password' in kwargs: + self._password = kwargs['password'] + + # database + if 'database' in kwargs: + self._database = kwargs['database'] + + # port + if 'port' in kwargs: + self._port = kwargs['port'] + + # config + if 'config' in kwargs: + self._config = kwargs['config'] + + self._chandle = CTaosInterface(self._config) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) + + def close(self): + """Close current connection. + """ + return CTaosInterface.close(self._conn) + + def subscribe(self, restart, topic, sql, interval): + """Create a subscription. + """ + if self._conn is None: + return None + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) + return TDengineSubscription(sub) + + def cursor(self): + """Return a new Cursor object using the connection. + """ + return TDengineCursor(self) + + def commit(self): + """Commit any pending transaction to the database. + + Since TDengine do not support transactions, the implement is void functionality. + """ + pass + + def rollback(self): + """Void functionality + """ + pass + + def clear_result_set(self): + """Clear unused result set on this connection. + """ + pass + + +if __name__ == "__main__": + conn = TDengineConnection(host='192.168.1.107') + conn.close() + print("Hello world") diff --git a/src/connector/python/osx/python3/taos/constants.py b/src/connector/python/osx/python3/taos/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..93466f5184a6bf37c2e1c915a00aa5c5e91d1801 --- /dev/null +++ b/src/connector/python/osx/python3/taos/constants.py @@ -0,0 +1,42 @@ +"""Constants in TDengine python +""" + +from .dbapi import * + + +class FieldType(object): + """TDengine Field Types + """ + # type_code + C_NULL = 0 + C_BOOL = 1 + C_TINYINT = 2 + C_SMALLINT = 3 + C_INT = 4 + C_BIGINT = 5 + C_FLOAT = 6 + C_DOUBLE = 7 + C_BINARY = 8 + C_TIMESTAMP = 9 + C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 + # NULL value definition + # NOTE: These values should change according to C definition in tsdb.h + C_BOOL_NULL = 0x02 + C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 + C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 + C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 + C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 + C_FLOAT_NULL = float('nan') + C_DOUBLE_NULL = float('nan') + C_BINARY_NULL = bytearray([int('0xff', 16)]) + # Timestamp precision definition + C_TIMESTAMP_MILLI = 0 + C_TIMESTAMP_MICRO = 1 diff --git a/src/connector/python/osx/python3/taos/cursor.py b/src/connector/python/osx/python3/taos/cursor.py new file mode 100644 index 0000000000000000000000000000000000000000..2e7c362d547973bc3e78f2eb57a33b7fa2d0635e --- /dev/null +++ b/src/connector/python/osx/python3/taos/cursor.py @@ -0,0 +1,294 @@ +from .cinterface import CTaosInterface +from .error import * +from .constants import FieldType +import threading + +# querySeqNum = 0 + + +class TDengineCursor(object): + """Database cursor which is used to manage the context of a fetch operation. + + Attributes: + .description: Read-only attribute consists of 7-item sequences: + + > name (mondatory) + > type_code (mondatory) + > display_size + > internal_size + > precision + > scale + > null_ok + + This attribute will be None for operations that do not return rows or + if the cursor has not had an operation invoked via the .execute*() method yet. + + .rowcount:This read-only attribute specifies the number of rows that the last + .execute*() produced (for DQL statements like SELECT) or affected + """ + + def __init__(self, connection=None): + self._description = [] + self._rowcount = -1 + self._connection = None + self._result = None + self._fields = None + self._block = None + self._block_rows = -1 + self._block_iter = 0 + self._affected_rows = 0 + self._logfile = "" + self._threadId = threading.get_ident() + + if connection is not None: + self._connection = connection + + def __iter__(self): + return self + + def __next__(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetch iterator") + + if self._block_rows <= self._block_iter: + block, self._block_rows = CTaosInterface.fetchRow( + self._result, self._fields) + if self._block_rows == 0: + raise StopIteration + self._block = list(map(tuple, zip(*block))) + self._block_iter = 0 + + data = self._block[self._block_iter] + self._block_iter += 1 + + return data + + @property + def description(self): + """Return the description of the object. + """ + return self._description + + @property + def rowcount(self): + """Return the rowcount of the object + """ + return self._rowcount + + @property + def affected_rows(self): + """Return the rowcount of insertion + """ + return self._affected_rows + + def callproc(self, procname, *args): + """Call a stored database procedure with the given name. + + Void functionality since no stored procedures. + """ + pass + + def log(self, logfile): + self._logfile = logfile + + def close(self): + """Close the cursor. + """ + if self._connection is None: + return False + + self._reset_result() + self._connection = None + + return True + + def execute(self, operation, params=None): + """Prepare and execute a database operation (query or command). + """ + # if threading.get_ident() != self._threadId: + # info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) + # raise OperationalError(info) + # print(info) + # return None + + if not operation: + return None + + if not self._connection: + # TODO : change the exception raised here + raise ProgrammingError("Cursor is not connected") + + self._reset_result() + + stmt = operation + if params is not None: + pass + + # global querySeqNum + # querySeqNum += 1 + # localSeqNum = querySeqNum # avoid raice condition + # print(" >> Exec Query ({}): {}".format(localSeqNum, str(stmt))) + self._result = CTaosInterface.query(self._connection._conn, stmt) + # print(" << Query ({}) Exec Done".format(localSeqNum)) + if (self._logfile): + with open(self._logfile, "a") as logfile: + logfile.write("%s;\n" % operation) + + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno == 0: + if CTaosInterface.fieldsCount(self._result) == 0: + self._affected_rows += CTaosInterface.affectedRows( + self._result) + return CTaosInterface.affectedRows(self._result) + else: + self._fields = CTaosInterface.useResult( + self._result) + return self._handle_result() + else: + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + + def executemany(self, operation, seq_of_parameters): + """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. + """ + pass + + def fetchone(self): + """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. + """ + pass + + def fetchmany(self): + pass + + def istype(self, col, dataType): + if (dataType.upper() == "BOOL"): + if (self._description[col][1] == FieldType.C_BOOL): + return True + if (dataType.upper() == "TINYINT"): + if (self._description[col][1] == FieldType.C_TINYINT): + return True + if (dataType.upper() == "TINYINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + return True + if (dataType.upper() == "SMALLINT"): + if (self._description[col][1] == FieldType.C_SMALLINT): + return True + if (dataType.upper() == "SMALLINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + return True + if (dataType.upper() == "INT"): + if (self._description[col][1] == FieldType.C_INT): + return True + if (dataType.upper() == "INT UNSIGNED"): + if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + return True + if (dataType.upper() == "BIGINT"): + if (self._description[col][1] == FieldType.C_BIGINT): + return True + if (dataType.upper() == "BIGINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): + return True + if (dataType.upper() == "FLOAT"): + if (self._description[col][1] == FieldType.C_FLOAT): + return True + if (dataType.upper() == "DOUBLE"): + if (self._description[col][1] == FieldType.C_DOUBLE): + return True + if (dataType.upper() == "BINARY"): + if (self._description[col][1] == FieldType.C_BINARY): + return True + if (dataType.upper() == "TIMESTAMP"): + if (self._description[col][1] == FieldType.C_TIMESTAMP): + return True + if (dataType.upper() == "NCHAR"): + if (self._description[col][1] == FieldType.C_NCHAR): + return True + + return False + + def fetchall_row(self): + """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. + """ + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetchall(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def nextset(self): + """ + """ + pass + + def setinputsize(self, sizes): + pass + + def setutputsize(self, size, column=None): + pass + + def _reset_result(self): + """Reset the result to unused version. + """ + self._description = [] + self._rowcount = -1 + if self._result is not None: + CTaosInterface.freeResult(self._result) + self._result = None + self._fields = None + self._block = None + self._block_rows = -1 + self._block_iter = 0 + self._affected_rows = 0 + + def _handle_result(self): + """Handle the return result from query. + """ + # if threading.get_ident() != self._threadId: + # info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) + # raise OperationalError(info) + # print(info) + # return None + + self._description = [] + for ele in self._fields: + self._description.append( + (ele['name'], ele['type'], None, None, None, None, False)) + + return self._result diff --git a/src/connector/python/osx/python3/taos/dbapi.py b/src/connector/python/osx/python3/taos/dbapi.py new file mode 100644 index 0000000000000000000000000000000000000000..594681ada953abf388e503c23199043cf686e1a3 --- /dev/null +++ b/src/connector/python/osx/python3/taos/dbapi.py @@ -0,0 +1,44 @@ +"""Type Objects and Constructors. +""" + +import time +import datetime + + +class DBAPITypeObject(object): + def __init__(self, *values): + self.values = values + + def __com__(self, other): + if other in self.values: + return 0 + if other < self.values: + return 1 + else: + return -1 + + +Date = datetime.date +Time = datetime.time +Timestamp = datetime.datetime + + +def DataFromTicks(ticks): + return Date(*time.localtime(ticks)[:3]) + + +def TimeFromTicks(ticks): + return Time(*time.localtime(ticks)[3:6]) + + +def TimestampFromTicks(ticks): + return Timestamp(*time.localtime(ticks)[:6]) + + +Binary = bytes + +# STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) +# BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) +# NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) +# DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/osx/python3/taos/error.py b/src/connector/python/osx/python3/taos/error.py new file mode 100644 index 0000000000000000000000000000000000000000..c584badce8320cd35dc81e8f6b613c56163b1a29 --- /dev/null +++ b/src/connector/python/osx/python3/taos/error.py @@ -0,0 +1,66 @@ +"""Python exceptions +""" + + +class Error(Exception): + def __init__(self, msg=None, errno=None): + self.msg = msg + self._full_msg = self.msg + self.errno = errno + + def __str__(self): + return self._full_msg + + +class Warning(Exception): + """Exception raised for important warnings like data truncations while inserting. + """ + pass + + +class InterfaceError(Error): + """Exception raised for errors that are related to the database interface rather than the database itself. + """ + pass + + +class DatabaseError(Error): + """Exception raised for errors that are related to the database. + """ + pass + + +class DataError(DatabaseError): + """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. + """ + pass + + +class OperationalError(DatabaseError): + """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer + """ + pass + + +class IntegrityError(DatabaseError): + """Exception raised when the relational integrity of the database is affected. + """ + pass + + +class InternalError(DatabaseError): + """Exception raised when the database encounters an internal error. + """ + pass + + +class ProgrammingError(DatabaseError): + """Exception raised for programming errors. + """ + pass + + +class NotSupportedError(DatabaseError): + """Exception raised in case a method or database API was used which is not supported by the database,. + """ + pass diff --git a/src/connector/python/osx/python3/taos/subscription.py b/src/connector/python/osx/python3/taos/subscription.py new file mode 100644 index 0000000000000000000000000000000000000000..270d9de09217fc58a389981a3542698dd1c0428a --- /dev/null +++ b/src/connector/python/osx/python3/taos/subscription.py @@ -0,0 +1,57 @@ +from .cinterface import CTaosInterface +from .error import * + + +class TDengineSubscription(object): + """TDengine subscription object + """ + + def __init__(self, sub): + self._sub = sub + + def consume(self): + """Consume rows of a subscription + """ + if self._sub is None: + raise OperationalError("Invalid use of consume") + + result, fields = CTaosInterface.consume(self._sub) + buffer = [[] for i in range(len(fields))] + while True: + block, num_of_fields = CTaosInterface.fetchBlock(result, fields) + if num_of_fields == 0: + break + for i in range(len(fields)): + buffer[i].extend(block[i]) + + self.fields = fields + return list(map(tuple, zip(*buffer))) + + def close(self, keepProgress=True): + """Close the Subscription. + """ + if self._sub is None: + return False + + CTaosInterface.unsubscribe(self._sub, keepProgress) + return True + + +if __name__ == '__main__': + from .connection import TDengineConnection + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") + + # Generate a cursor object to run SQL commands + sub = conn.subscribe(True, "test", "select * from meters;", 1000) + + for i in range(0, 10): + data = sub.consume() + for d in data: + print(d) + + sub.close() + conn.close() diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index 5ddbe83011daa9d408e22dbeda7172466329921e..333f5bedad7edeefab66e1c1ba90424085e2aa1c 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.4", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py index 62e0536b6fca63a0aca0b6be3673104c27c631c6..973263573808232e4e71dc0158585624a8e7d2ab 100644 --- a/src/connector/python/windows/python2/taos/__init__.py +++ b/src/connector/python/windows/python2/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index 14f4f49be8ae719da16dbed7e79631db7ef9689e..d8cdce2ad138c34db5193e3972ba51d46c693254 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,168 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) - return res + return res + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.windll.LoadLibrary('taos') @@ -216,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -227,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -236,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -255,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -268,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -293,12 +443,13 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -308,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -360,35 +511,49 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) diff --git a/src/connector/python/windows/python2/taos/connection.py b/src/connector/python/windows/python2/taos/connection.py index d9576a553b810a975429b2cefc03e5e60f240a88..5729d01c6df8c0e58086726c4001467811e9fee5 100644 --- a/src/connector/python/windows/python2/taos/connection.py +++ b/src/connector/python/windows/python2/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -30,7 +32,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -44,7 +46,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -56,7 +63,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -81,7 +89,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/windows/python2/taos/constants.py b/src/connector/python/windows/python2/taos/constants.py index a994bceaf61894ac0bf9a719a574d00a09c584a5..8a8011c3e36c52993e9d03228c2a50e2af6a7c9e 100644 --- a/src/connector/python/windows/python2/taos/constants.py +++ b/src/connector/python/windows/python2/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index f6fde2619b0a727c74904c2338c813b97a2e602c..0656b6326e173b111eb8293c6e3b76678eccc0e2 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -5,6 +5,7 @@ import threading # querySeqNum = 0 + class TDengineCursor(object): """Database cursor which is used to manage the context of a fetch operation. @@ -23,7 +24,7 @@ class TDengineCursor(object): if the cursor has not had an operation invoked via the .execute*() method yet. .rowcount:This read-only attribute specifies the number of rows that the last - .execute*() produced (for DQL statements like SELECT) or affected + .execute*() produced (for DQL statements like SELECT) or affected """ def __init__(self, connection=None): @@ -50,13 +51,14 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow( + self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) self._block_iter = 0 - data = self._block[self._block_iter] + data = self._block[self._block_iter] self._block_iter += 1 return data @@ -91,7 +93,7 @@ class TDengineCursor(object): """ if self._connection is None: return False - + self._reset_result() self._connection = None @@ -106,19 +108,20 @@ class TDengineCursor(object): if not self._connection: # TODO : change the exception raised here raise ProgrammingError("Cursor is not connected") - + self._reset_result() stmt = operation if params is not None: pass - + self._result = CTaosInterface.query(self._connection._conn, stmt) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: - self._affected_rows += CTaosInterface.affectedRows(self._result) - return CTaosInterface.affectedRows(self._result ) + self._affected_rows += CTaosInterface.affectedRows( + self._result) + return CTaosInterface.affectedRows(self._result) else: self._fields = CTaosInterface.useResult(self._result) return self._handle_result() @@ -138,7 +141,7 @@ class TDengineCursor(object): def fetchmany(self): pass - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -147,38 +150,42 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - - def fetchall_block(self): + + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) - if num_of_fields == 0: break + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) - return list(map(tuple, zip(*buffer))) - - def nextset(self): """ """ @@ -209,6 +216,7 @@ class TDengineCursor(object): """ self._description = [] for ele in self._fields: - self._description.append((ele['name'], ele['type'], None, None, None, None, False)) - + self._description.append( + (ele['name'], ele['type'], None, None, None, None, False)) + return self._result diff --git a/src/connector/python/windows/python2/taos/dbapi.py b/src/connector/python/windows/python2/taos/dbapi.py index f1c22bdb512224ac712b78b15ec00207587e65c5..594681ada953abf388e503c23199043cf686e1a3 100644 --- a/src/connector/python/windows/python2/taos/dbapi.py +++ b/src/connector/python/windows/python2/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/windows/python2/taos/error.py b/src/connector/python/windows/python2/taos/error.py index 24508a72ed78bb6231187bb6de34d57182e31b22..c584badce8320cd35dc81e8f6b613c56163b1a29 100644 --- a/src/connector/python/windows/python2/taos/error.py +++ b/src/connector/python/windows/python2/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/windows/python2/taos/subscription.py b/src/connector/python/windows/python2/taos/subscription.py index d3cf10d5ada578687689b94454378dd543368e3e..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/windows/python2/taos/subscription.py +++ b/src/connector/python/windows/python2/taos/subscription.py @@ -1,32 +1,33 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: @@ -38,15 +39,19 @@ class TDengineSubscription(object): if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index ffed304c855fda28287735d0daab27e17b0e976e..f29fec121b27f5e87ffd215bb4ff5006acf3bf1c 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.4", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py index c6dd929a6a5110300ecd2e1042d48a5ad6a4bf46..b57e25fd2c320956e46b190d9f0a1139db1cced0 100644 --- a/src/connector/python/windows/python3/taos/__init__.py +++ b/src/connector/python/windows/python3/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index 42b820ca80feda4a269e5e80f288bd1f5e87adcd..d8cdce2ad138c34db5193e3972ba51d46c693254 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,170 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ulong))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res - + + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.windll.LoadLibrary('taos') @@ -218,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -229,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -238,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -257,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -270,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -295,7 +443,8 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: @@ -310,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -362,35 +511,49 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) @@ -476,4 +639,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/windows/python3/taos/connection.py b/src/connector/python/windows/python3/taos/connection.py index d9576a553b810a975429b2cefc03e5e60f240a88..5729d01c6df8c0e58086726c4001467811e9fee5 100644 --- a/src/connector/python/windows/python3/taos/connection.py +++ b/src/connector/python/windows/python3/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -30,7 +32,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -44,7 +46,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -56,7 +63,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -81,7 +89,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/windows/python3/taos/constants.py b/src/connector/python/windows/python3/taos/constants.py index def2bbc0a8c063e85214a634b60c5db7a5fd1259..49fc17b2fb98a6684e74e4a044651fdc6237518e 100644 --- a/src/connector/python/windows/python3/taos/constants.py +++ b/src/connector/python/windows/python3/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index db66b99d3bfe5935a3e5b04c3692453092bcf24e..769cb7cf0f61fe850c16315bf552162f33536502 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -24,7 +24,7 @@ class TDengineCursor(object): if the cursor has not had an operation invoked via the .execute*() method yet. .rowcount:This read-only attribute specifies the number of rows that the last - .execute*() produced (for DQL statements like SELECT) or affected + .execute*() produced (for DQL statements like SELECT) or affected """ def __init__(self, connection=None): @@ -51,13 +51,14 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow( + self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) self._block_iter = 0 - data = self._block[self._block_iter] + data = self._block[self._block_iter] self._block_iter += 1 return data @@ -92,7 +93,7 @@ class TDengineCursor(object): """ if self._connection is None: return False - + self._reset_result() self._connection = None @@ -107,24 +108,25 @@ class TDengineCursor(object): if not self._connection: # TODO : change the exception raised here raise ProgrammingError("Cursor is not connected") - + self._reset_result() stmt = operation if params is not None: pass - + self._result = CTaosInterface.query(self._connection._conn, stmt) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: - self._affected_rows += CTaosInterface.affectedRows(self._result ) - return CTaosInterface.affectedRows(self._result ) + self._affected_rows += CTaosInterface.affectedRows( + self._result) + return CTaosInterface.affectedRows(self._result) else: - self._fields = CTaosInterface.useResult(self._result ) + self._fields = CTaosInterface.useResult(self._result) return self._handle_result() else: - raise ProgrammingError(CTaosInterface.errStr(self._result ), errno) + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) def executemany(self, operation, seq_of_parameters): """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. @@ -139,7 +141,7 @@ class TDengineCursor(object): def fetchmany(self): pass - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -148,10 +150,13 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -159,27 +164,28 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def fetchall_block(self): + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) - if num_of_fields == 0: break + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) - return list(map(tuple, zip(*buffer))) - - def nextset(self): """ """ @@ -204,12 +210,13 @@ class TDengineCursor(object): self._block_rows = -1 self._block_iter = 0 self._affected_rows = 0 - + def _handle_result(self): """Handle the return result from query. """ self._description = [] for ele in self._fields: - self._description.append((ele['name'], ele['type'], None, None, None, None, False)) - + self._description.append( + (ele['name'], ele['type'], None, None, None, None, False)) + return self._result diff --git a/src/connector/python/windows/python3/taos/dbapi.py b/src/connector/python/windows/python3/taos/dbapi.py index 9b1cb1321c14619782c801e9381010f1f67fbc2e..a29621f7a3594a618b59b30bdc96197c4222a619 100644 --- a/src/connector/python/windows/python3/taos/dbapi.py +++ b/src/connector/python/windows/python3/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/windows/python3/taos/error.py b/src/connector/python/windows/python3/taos/error.py index ccc0e61d84597a7cae9b360b7fa2434b9bc47401..238b293a0b609570e7b5d536648c6ada3ca2f209 100644 --- a/src/connector/python/windows/python3/taos/error.py +++ b/src/connector/python/windows/python3/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/windows/python3/taos/subscription.py b/src/connector/python/windows/python3/taos/subscription.py index d3cf10d5ada578687689b94454378dd543368e3e..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/windows/python3/taos/subscription.py +++ b/src/connector/python/windows/python3/taos/subscription.py @@ -1,32 +1,33 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: @@ -38,15 +39,19 @@ class TDengineSubscription(object): if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index e6313973484467e4b67be88ffc21225923739b71..e9ed2996c74e2c59d56245e6fc1e932ebb07dfb0 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -6,12 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) - ADD_LIBRARY(tcq ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) - ENDIF () - ADD_SUBDIRECTORY(test) +ADD_LIBRARY(tcq ${SRC}) +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(tcq tutil common taos_static) +ELSE () + TARGET_LINK_LIBRARIES(tcq tutil common taos) ENDIF () +ADD_SUBDIRECTORY(test) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index efb8795962e769dd0b11b8d313177855c994f52d..fb0c1508cb051c577fed9bde534d55e297822fe2 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -43,20 +43,24 @@ typedef struct { int32_t master; int32_t num; // number of continuous streams char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; + char pass[TSDB_KEY_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; struct SCqObj *pHead; void *dbConn; void *tmrCtrl; pthread_mutex_t mutex; + int32_t delete; + int32_t cqObjNum; } SCqContext; typedef struct SCqObj { tmr_h tmrId; + int64_t rid; uint64_t uid; int32_t tid; // table ID int32_t rowSize; // bytes of a row + char * dstTable; char * sqlStr; // SQL string STSchema * pSchema; // pointer to schema array void * pStream; @@ -68,6 +72,90 @@ typedef struct SCqObj { static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); +int32_t cqObjRef = -1; + +void cqRmFromList(SCqObj *pObj) { + //LOCK in caller + + SCqContext *pContext = pObj->pContext; + + if (pObj->prev) { + pObj->prev->next = pObj->next; + } else { + pContext->pHead = pObj->next; + } + + if (pObj->next) { + pObj->next->prev = pObj->prev; + } + +} + +static void freeSCqContext(void *handle) { + if (handle == NULL) { + return; + } + SCqContext *pContext = handle; + pthread_mutex_destroy(&pContext->mutex); + + taosTmrCleanUp(pContext->tmrCtrl); + pContext->tmrCtrl = NULL; + cDebug("vgId:%d, CQ is closed", pContext->vgId); + free(pContext); +} + + +void cqFree(void *handle) { + if (tsEnableStream == 0) { + return; + } + SCqObj *pObj = handle; + SCqContext *pContext = pObj->pContext; + int32_t delete = 0; + + pthread_mutex_lock(&pContext->mutex); + + // free the resources associated + if (pObj->pStream) { + taos_close_stream(pObj->pStream); + pObj->pStream = NULL; + } else { + taosTmrStop(pObj->tmrId); + pObj->tmrId = 0; + } + + cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); + tdFreeSchema(pObj->pSchema); + free(pObj->dstTable); + free(pObj->sqlStr); + free(pObj); + + pContext->cqObjNum--; + + if (pContext->cqObjNum <= 0 && pContext->delete) { + delete = 1; + } + + pthread_mutex_unlock(&pContext->mutex); + + if (delete) { + freeSCqContext(pContext); + } +} + + +void cqCreateRef() { + int32_t ref = atomic_load_32(&cqObjRef); + if (ref == -1) { + ref = taosOpenRef(4096, cqFree); + + if (atomic_val_compare_exchange_32(&cqObjRef, -1, ref) != -1) { + taosCloseRef(ref); + } + } +} + + void *cqOpen(void *ahandle, const SCqCfg *pCfg) { if (tsEnableStream == 0) { return NULL; @@ -78,6 +166,8 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { return NULL; } + cqCreateRef(); + pContext->tmrCtrl = taosTmrInit(0, 0, 0, "CQ"); tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user)); @@ -96,11 +186,13 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { pthread_mutex_init(&pContext->mutex, NULL); - cInfo("vgId:%d, CQ is opened", pContext->vgId); + + cDebug("vgId:%d, CQ is opened", pContext->vgId); return pContext; } + void cqClose(void *handle) { if (tsEnableStream == 0) { return; @@ -108,30 +200,46 @@ void cqClose(void *handle) { SCqContext *pContext = handle; if (handle == NULL) return; + pContext->delete = 1; + int32_t hasCq = 0; + int32_t existLoop = 0; + // stop all CQs cqStop(pContext); - // free all resources - pthread_mutex_lock(&pContext->mutex); + int64_t rid = 0; - SCqObj *pObj = pContext->pHead; - while (pObj) { - SCqObj *pTemp = pObj; - pObj = pObj->next; - tdFreeSchema(pTemp->pSchema); - tfree(pTemp->sqlStr); - free(pTemp); - } - - pthread_mutex_unlock(&pContext->mutex); + while (1) { + pthread_mutex_lock(&pContext->mutex); - pthread_mutex_destroy(&pContext->mutex); + SCqObj *pObj = pContext->pHead; + if (pObj) { + cqRmFromList(pObj); - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; + rid = pObj->rid; - cInfo("vgId:%d, CQ is closed", pContext->vgId); - free(pContext); + hasCq = 1; + + if (pContext->pHead == NULL) { + existLoop = 1; + } + } else { + pthread_mutex_unlock(&pContext->mutex); + break; + } + + pthread_mutex_unlock(&pContext->mutex); + + taosRemoveRef(cqObjRef, rid); + + if (existLoop) { + break; + } + } + + if (hasCq == 0) { + freeSCqContext(pContext); + } } void cqStart(void *handle) { @@ -141,7 +249,7 @@ void cqStart(void *handle) { SCqContext *pContext = handle; if (pContext->dbConn || pContext->master) return; - cInfo("vgId:%d, start all CQs", pContext->vgId); + cDebug("vgId:%d, start all CQs", pContext->vgId); pthread_mutex_lock(&pContext->mutex); pContext->master = 1; @@ -159,8 +267,9 @@ void cqStop(void *handle) { if (tsEnableStream == 0) { return; } + SCqContext *pContext = handle; - cInfo("vgId:%d, stop all CQs", pContext->vgId); + cDebug("vgId:%d, stop all CQs", pContext->vgId); if (pContext->dbConn == NULL || pContext->master == 0) return; pthread_mutex_lock(&pContext->mutex); @@ -185,19 +294,22 @@ void cqStop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) { +void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema) { if (tsEnableStream == 0) { return NULL; } SCqContext *pContext = handle; - + int64_t rid = 0; + SCqObj *pObj = calloc(sizeof(SCqObj), 1); if (pObj == NULL) return NULL; pObj->uid = uid; - pObj->tid = tid; - pObj->sqlStr = malloc(strlen(sqlStr)+1); - strcpy(pObj->sqlStr, sqlStr); + pObj->tid = sid; + if (dstTable != NULL) { + pObj->dstTable = strdup(dstTable); + } + pObj->sqlStr = strdup(sqlStr); pObj->pSchema = tdDupSchema(pSchema); pObj->rowSize = schemaTLen(pSchema); @@ -210,32 +322,36 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema * if (pContext->pHead) pContext->pHead->prev = pObj; pContext->pHead = pObj; + pContext->cqObjNum++; + + pObj->rid = taosAddRef(cqObjRef, pObj); + cqCreateStream(pContext, pObj); + rid = pObj->rid; + pthread_mutex_unlock(&pContext->mutex); - return pObj; + + return (void *)rid; } void cqDrop(void *handle) { if (tsEnableStream == 0) { return; } - SCqObj *pObj = handle; - SCqContext *pContext = pObj->pContext; - - pthread_mutex_lock(&pContext->mutex); - - if (pObj->prev) { - pObj->prev->next = pObj->next; - } else { - pContext->pHead = pObj->next; - } - if (pObj->next) { - pObj->next->prev = pObj->prev; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)handle); + if (pObj == NULL) { + return; } + + SCqContext *pContext = pObj->pContext; + + pthread_mutex_lock(&pContext->mutex); + cqRmFromList(pObj); + // free the resources associated if (pObj->pStream) { taos_close_stream(pObj->pStream); @@ -245,16 +361,18 @@ void cqDrop(void *handle) { pObj->tmrId = 0; } - cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); - tdFreeSchema(pObj->pSchema); - free(pObj->sqlStr); - free(pObj); - pthread_mutex_unlock(&pContext->mutex); + + taosRemoveRef(cqObjRef, (int64_t)handle); + taosReleaseRef(cqObjRef, (int64_t)handle); } static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { - SCqObj* pObj = (SCqObj*)param; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); + if (pObj == NULL) { + return; + } + SCqContext* pContext = pObj->pContext; SSqlObj* pSql = (SSqlObj*)result; if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) { @@ -263,10 +381,16 @@ static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { pthread_mutex_lock(&pContext->mutex); cqCreateStream(pContext, pObj); pthread_mutex_unlock(&pContext->mutex); + + taosReleaseRef(cqObjRef, (int64_t)param); } static void cqProcessCreateTimer(void *param, void *tmrId) { - SCqObj* pObj = (SCqObj*)param; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); + if (pObj == NULL) { + return; + } + SCqContext* pContext = pObj->pContext; if (pContext->dbConn == NULL) { @@ -277,6 +401,8 @@ static void cqProcessCreateTimer(void *param, void *tmrId) { cqCreateStream(pContext, pObj); pthread_mutex_unlock(&pContext->mutex); } + + taosReleaseRef(cqObjRef, (int64_t)param); } static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { @@ -284,16 +410,19 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { if (pContext->dbConn == NULL) { cDebug("vgId:%d, create dbConn after 1000 ms", pContext->vgId); - pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, pObj, pContext->tmrCtrl); + pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, (void *)pObj->rid, pContext->tmrCtrl); return; } pObj->tmrId = 0; if (pObj->pStream == NULL) { - pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL); + pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, (void *)pObj->rid, NULL); + + // TODO the pObj->pStream may be released if error happens if (pObj->pStream) { + tscSetStreamDestTable(pObj->pStream, pObj->dstTable); pContext->num++; - cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr); + cDebug("vgId:%d, id:%d CQ:%s is opened", pContext->vgId, pObj->tid, pObj->sqlStr); } else { cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr); } @@ -301,15 +430,28 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { } static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { - SCqObj *pObj = (SCqObj *)param; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); + if (pObj == NULL) { + return; + } + if (tres == NULL && row == NULL) { + taos_close_stream(pObj->pStream); + pObj->pStream = NULL; + + taosReleaseRef(cqObjRef, (int64_t)param); + return; } + SCqContext *pContext = pObj->pContext; STSchema *pSchema = pObj->pSchema; - if (pObj->pStream == NULL) return; - + if (pObj->pStream == NULL) { + taosReleaseRef(cqObjRef, (int64_t)param); + return; + } + cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; @@ -331,9 +473,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { val = ((char*)val) - sizeof(VarDataLenT); } else if (c->type == TSDB_DATA_TYPE_NCHAR) { char buf[TSDB_MAX_NCHAR_LEN]; - size_t len = taos_fetch_lengths(tres)[i]; + int32_t len = taos_fetch_lengths(tres)[i]; taosMbsToUcs4(val, len, buf, sizeof(buf), &len); - memcpy(val + sizeof(VarDataLenT), buf, len); + memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); @@ -360,5 +502,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { // write into vnode write queue pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL); free(buffer); + + taosReleaseRef(cqObjRef, (int64_t)param); } diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt index 59c0ac79cfffd35a40beb58a2a172a9bf27dca15..cd124567afd8766173cf07e7a6191ab473be1714 100644 --- a/src/cq/test/CMakeLists.txt +++ b/src/cq/test/CMakeLists.txt @@ -3,4 +3,4 @@ PROJECT(TDengine) LIST(APPEND CQTEST_SRC ./cqtest.c) ADD_EXECUTABLE(cqtest ${CQTEST_SRC}) -TARGET_LINK_LIBRARIES(cqtest tcq) +TARGET_LINK_LIBRARIES(cqtest tcq taos_static) diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index 41380f0d86ed31b2466596993b8ce1a1d435f8cf..f378835f0a87f9dd3539d92ac052c81cabcc165d 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -70,7 +70,7 @@ int main(int argc, char *argv[]) { tdDestroyTSchemaBuilder(&schemaBuilder); for (int sid =1; sid<10; ++sid) { - cqCreate(pCq, sid, sid, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); + cqCreate(pCq, sid, sid, NULL, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); } tdFreeSchema(pSchema); diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 5608cfd6d16826e399f9ea41dc026d2ab2459610..644a4e875d62622c07034639a4e08e584e99fdfb 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -10,44 +10,39 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) - ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosd taos_static) - ELSE () - TARGET_LINK_LIBRARIES(taosd taos) - ENDIF () - - IF (TD_ACCOUNT) - TARGET_LINK_LIBRARIES(taosd account) - ENDIF () - - IF (TD_GRANT) - TARGET_LINK_LIBRARIES(taosd grant) - ENDIF () - - IF (TD_MQTT) - TARGET_LINK_LIBRARIES(taosd mqtt) - ENDIF () - - IF (TD_SYNC) - TARGET_LINK_LIBRARIES(taosd balance sync) - ENDIF () - - SET(PREPARE_ENV_CMD "prepare_env_cmd") - SET(PREPARE_ENV_TARGET "prepare_env_target") - ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - POST_BUILD - COMMAND echo "make test directory" - DEPENDS taosd - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMENT "prepare taosd environment") - ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +ADD_EXECUTABLE(taosd ${SRC}) +TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) + +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosd taos_static) +ELSE () + TARGET_LINK_LIBRARIES(taosd taos) +ENDIF () + +IF (TD_ACCOUNT) + TARGET_LINK_LIBRARIES(taosd account) +ENDIF () + +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(taosd grant) +ENDIF () + +IF (TD_MQTT) + TARGET_LINK_LIBRARIES(taosd mqtt) ENDIF () + +SET(PREPARE_ENV_CMD "prepare_env_cmd") +SET(PREPARE_ENV_TARGET "prepare_env_target") +ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} + POST_BUILD + COMMAND echo "make test directory" + DEPENDS taosd + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ + COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo monitor 0 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMENT "prepare taosd environment") +ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h index 35d889646021a079abe18c01a26fa5891718aaa0..896b3f574c2e0d02a0d62048a411fa484d16130b 100644 --- a/src/dnode/inc/dnodeCfg.h +++ b/src/dnode/inc/dnodeCfg.h @@ -19,11 +19,13 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitCfg(); void dnodeCleanupCfg(); void dnodeUpdateCfg(SDnodeCfg *cfg); int32_t dnodeGetDnodeId(); +void dnodeGetClusterId(char *clusterId); void dnodeGetCfg(int32_t *dnodeId, char *clusterId); #ifdef __cplusplus diff --git a/src/dnode/inc/dnodeCheck.h b/src/dnode/inc/dnodeCheck.h index a4880b3c114342c8900337372bd1dfd07e7815b7..c94b9e93197f4bfc9a175ee5825fd2c1c25d7072 100644 --- a/src/dnode/inc/dnodeCheck.h +++ b/src/dnode/inc/dnodeCheck.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitCheck(); void dnodeCleanupCheck(); diff --git a/src/dnode/inc/dnodeEps.h b/src/dnode/inc/dnodeEps.h index 2a203498c1f270c7e456694a4e1e195cbd9022cd..a5840997b0f9623c8ed504fa1c2eb8209216ea29 100644 --- a/src/dnode/inc/dnodeEps.h +++ b/src/dnode/inc/dnodeEps.h @@ -19,8 +19,7 @@ #ifdef __cplusplus extern "C" { #endif - -#include "taosmsg.h" +#include "dnodeInt.h" int32_t dnodeInitEps(); void dnodeCleanupEps(); diff --git a/src/dnode/inc/dnodeInt.h b/src/dnode/inc/dnodeInt.h index f4cbee1d1333e3f1ffc213f9e2ecdeb5f4c32d57..1327cd4433fd2e2157becaaf5cb52e2ca0ffe6ef 100644 --- a/src/dnode/inc/dnodeInt.h +++ b/src/dnode/inc/dnodeInt.h @@ -19,8 +19,13 @@ #ifdef __cplusplus extern "C" { #endif - +#include "taoserror.h" +#include "taosmsg.h" #include "tlog.h" +#include "trpc.h" +#include "tglobal.h" +#include "dnode.h" +#include "vnode.h" extern int32_t dDebugFlag; @@ -31,6 +36,14 @@ extern int32_t dDebugFlag; #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} +typedef enum { + TSDB_RUN_STATUS_INITIALIZE, + TSDB_RUN_STATUS_RUNING, + TSDB_RUN_STATUS_STOPPED +} SRunStatus; + +SRunStatus dnodeGetRunStatus(); + #ifdef __cplusplus } #endif diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h index 2c3eef5d5d98e43db1ae783f9e7db2769fcf521e..f05e2b6f7b710f9ebe547e27bf524c13ffcfaea8 100644 --- a/src/dnode/inc/dnodeMInfos.h +++ b/src/dnode/inc/dnodeMInfos.h @@ -19,8 +19,7 @@ #ifdef __cplusplus extern "C" { #endif - -#include "taosmsg.h" +#include "dnodeInt.h" int32_t dnodeInitMInfos(); void dnodeCleanupMInfos(); @@ -29,6 +28,10 @@ void dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet); void dnodeGetMInfos(SMInfos *pMinfos); bool dnodeIsMasterEp(char *ep); +void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); +void dnodeGetEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetEpSetForShell(SRpcEpSet *epSet); + #ifdef __cplusplus } #endif diff --git a/src/dnode/inc/dnodeMPeer.h b/src/dnode/inc/dnodeMPeer.h index 00221baa221a411d614c1fa1bb3dc4525de2ed71..b7e566d7e4952d6afbd88e64b36197de4f0f4401 100644 --- a/src/dnode/inc/dnodeMPeer.h +++ b/src/dnode/inc/dnodeMPeer.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitMPeer(); void dnodeCleanupMPeer(); diff --git a/src/dnode/inc/dnodeMRead.h b/src/dnode/inc/dnodeMRead.h index 8a8e71227ddf3cd7b7c77ef664ac3d4c935889db..279098d30e137bc6b57ef09ce93b8d12fd421082 100644 --- a/src/dnode/inc/dnodeMRead.h +++ b/src/dnode/inc/dnodeMRead.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitMRead(); void dnodeCleanupMRead(); diff --git a/src/dnode/inc/dnodeMWrite.h b/src/dnode/inc/dnodeMWrite.h index 6a3d41bc81a03465a2e097bc711c83a8d8af362e..8d4fcce3befbad35b47b4817c02662adc37addf8 100644 --- a/src/dnode/inc/dnodeMWrite.h +++ b/src/dnode/inc/dnodeMWrite.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitMWrite(); void dnodeCleanupMWrite(); diff --git a/src/dnode/inc/dnodeMain.h b/src/dnode/inc/dnodeMain.h index c1480407bde0bbcf7aada20f90084b96fc380a73..ca79d53afdc0b211f3903b9323f7e567d0045093 100644 --- a/src/dnode/inc/dnodeMain.h +++ b/src/dnode/inc/dnodeMain.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitSystem(); void dnodeCleanUpSystem(); diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h index 8618de324446ca3f2504d15a6422bcca3a4b51b0..863ea433c46c175ee7cb6ad46e6dd38f164d36b1 100644 --- a/src/dnode/inc/dnodeModule.h +++ b/src/dnode/inc/dnodeModule.h @@ -19,11 +19,11 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitModules(); -void dnodeStartModules(); void dnodeCleanupModules(); -void dnodeProcessModuleStatus(uint32_t moduleStatus); +int32_t dnodeStartMnode(SMInfos *pMinfos); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodePeer.h b/src/dnode/inc/dnodePeer.h index 0dcf48f2322be91de792148165b2a127fa983196..6d337ef6dc65d717c382ebaf59a50cf334686a64 100644 --- a/src/dnode/inc/dnodePeer.h +++ b/src/dnode/inc/dnodePeer.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitServer(); void dnodeCleanupServer(); diff --git a/src/dnode/inc/dnodeShell.h b/src/dnode/inc/dnodeShell.h index 300c86c5999553b2332b3224fc6c65ef019dc5af..3fa66d6a3bc24d253ede6076f9b6bdacdb14db7d 100644 --- a/src/dnode/inc/dnodeShell.h +++ b/src/dnode/inc/dnodeShell.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitShell(); void dnodeCleanupShell(); diff --git a/src/dnode/inc/dnodeStep.h b/src/dnode/inc/dnodeStep.h new file mode 100644 index 0000000000000000000000000000000000000000..e181e19c4633aa6c844d92db15cee1a9930a41ac --- /dev/null +++ b/src/dnode/inc/dnodeStep.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_STEP_H +#define TDENGINE_DNODE_STEP_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize); +void dnodeStepCleanup(SStep *pSteps, int32_t stepSize); +void dnodeReportStep(char *name, char *desc, int8_t finished); +void dnodeSendStartupStep(SRpcMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/dnode/inc/dnodeTelemetry.h b/src/dnode/inc/dnodeTelemetry.h index 6fb62556ae50ec3759c7dc2260d16f36d1daf025..e4fd5a0376e3e7fdca0bbe1be15fa1cd70fd7695 100644 --- a/src/dnode/inc/dnodeTelemetry.h +++ b/src/dnode/inc/dnodeTelemetry.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitTelemetry(); void dnodeCleanupTelemetry(); diff --git a/src/dnode/inc/dnodeVMgmt.h b/src/dnode/inc/dnodeVMgmt.h new file mode 100644 index 0000000000000000000000000000000000000000..821196defc9b827961ec4ae8140be82186192f62 --- /dev/null +++ b/src/dnode/inc/dnodeVMgmt.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_VMGMT_H +#define TDENGINE_DNODE_VMGMT_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +int32_t dnodeInitVMgmt(); +void dnodeCleanupVMgmt(); +void dnodeDispatchToVMgmtQueue(SRpcMsg *rpcMsg); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/dnode/inc/dnodeVRead.h b/src/dnode/inc/dnodeVRead.h index 5b17693146cb68ab66943593bc2f9cd6587cc225..9c88886f88bedaa63a5071b9dd2d773d4ff1cc0c 100644 --- a/src/dnode/inc/dnodeVRead.h +++ b/src/dnode/inc/dnodeVRead.h @@ -19,12 +19,15 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitVRead(); void dnodeCleanupVRead(); void dnodeDispatchToVReadQueue(SRpcMsg *pMsg); -void * dnodeAllocVReadQueue(void *pVnode); -void dnodeFreeVReadQueue(void *pRqueue); +void * dnodeAllocVQueryQueue(void *pVnode); +void * dnodeAllocVFetchQueue(void *pVnode); +void dnodeFreeVQueryQueue(void *pQqueue); +void dnodeFreeVFetchQueue(void *pFqueue); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeVWrite.h b/src/dnode/inc/dnodeVWrite.h index 759e9ca8a5599236d08228ddd87ed8d4f8c55dca..2ddff210f8620cfe0544cb5925f633cab8d496af 100644 --- a/src/dnode/inc/dnodeVWrite.h +++ b/src/dnode/inc/dnodeVWrite.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "dnodeInt.h" int32_t dnodeInitVWrite(); void dnodeCleanupVWrite(); diff --git a/src/dnode/inc/dnodeVnodes.h b/src/dnode/inc/dnodeVnodes.h new file mode 100644 index 0000000000000000000000000000000000000000..e60dd290ce478f214c26410194f345fd4840dfb8 --- /dev/null +++ b/src/dnode/inc/dnodeVnodes.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_VNODES_H +#define TDENGINE_DNODE_VNODES_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +int32_t dnodeInitVnodes(); +void dnodeCleanupVnodes(); +int32_t dnodeInitStatusTimer(); +void dnodeCleanupStatusTimer(); +void dnodeSendStatusMsgToMnode(); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c index 16d109a13a7600c5414b1f1edd175a8f15aa4cf0..fd5956b37f28603893b43ca91f8cfd1177dccad2 100644 --- a/src/dnode/src/dnodeCfg.c +++ b/src/dnode/src/dnodeCfg.c @@ -16,9 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "cJSON.h" -#include "tglobal.h" -#include "dnode.h" -#include "dnodeInt.h" #include "dnodeCfg.h" static SDnodeCfg tsCfg = {0}; @@ -54,6 +51,12 @@ int32_t dnodeGetDnodeId() { return dnodeId; } +void dnodeGetClusterId(char *clusterId) { + pthread_mutex_lock(&tsCfgMutex); + tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&tsCfgMutex); +} + void dnodeGetCfg(int32_t *dnodeId, char *clusterId) { pthread_mutex_lock(&tsCfgMutex); *dnodeId = tsCfg.dnodeId; @@ -94,7 +97,7 @@ static int32_t dnodeReadCfg() { goto PARSE_CFG_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_CFG_OVER; @@ -112,7 +115,7 @@ static int32_t dnodeReadCfg() { dError("failed to read %s, dnodeId not found", file); goto PARSE_CFG_OVER; } - cfg.dnodeId = dnodeId->valueint; + cfg.dnodeId = (int32_t)dnodeId->valueint; cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { @@ -153,7 +156,7 @@ static int32_t dnodeWriteCfg() { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index a9ee4ac649c2f4b2734ce7f0dd59004b08e2fb67..87baff30673afc68eb23a00bef279433a422ba67 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -15,8 +15,6 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tglobal.h" -#include "dnodeInt.h" #include "dnodeCheck.h" typedef struct { @@ -31,8 +29,8 @@ typedef struct { static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; -static int bindTcpPort(int port) { - int serverSocket; +static int32_t bindTcpPort(uint16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { @@ -47,22 +45,22 @@ static int bindTcpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d tcp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } if (listen(serverSocket, 5) < 0) { dError("port:%d listen() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int bindUdpPort(int port) { - int serverSocket; +static int32_t bindUdpPort(int16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { @@ -77,19 +75,19 @@ static int bindUdpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d udp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int dnodeCheckNetwork() { - int ret; - int startPort = tsServerPort; +static int32_t dnodeCheckNetwork() { + int32_t ret; + uint16_t startPort = tsServerPort; - for (int port = startPort; port < startPort + 12; port++) { + for (uint16_t port = startPort; port < startPort + 12; port++) { ret = bindTcpPort(port); if (0 != ret) { dError("failed to tcp bind port %d, quit", port); @@ -105,7 +103,7 @@ static int dnodeCheckNetwork() { return 0; } -static int dnodeCheckMem() { +static int32_t dnodeCheckMem() { float memoryUsedMB; float memoryAvailMB; if (true != taosGetSysMemory(&memoryUsedMB)) { @@ -123,12 +121,12 @@ static int dnodeCheckMem() { return 0; } -static int dnodeCheckCpu() { +static int32_t dnodeCheckCpu() { // TODO: return 0; } -static int dnodeCheckDisk() { +static int32_t dnodeCheckDisk() { taosGetDisk(); if (tsAvailDataDirGB < tsMinimalDataDirGB) { @@ -149,24 +147,24 @@ static int dnodeCheckDisk() { return 0; } -static int dnodeCheckOs() { +static int32_t dnodeCheckOs() { // TODO: return 0; } -static int dnodeCheckAccess() { +static int32_t dnodeCheckAccess() { // TODO: return 0; } -static int dnodeCheckVersion() { +static int32_t dnodeCheckVersion() { // TODO: return 0; } -static int dnodeCheckDatafile() { +static int32_t dnodeCheckDatafile() { // TODO: return 0; diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index 83f294e05eb0e559638fef1259ad96ac6f162460..9554651776ad0e9c961a3d15955491b224d869bf 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -16,10 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "cJSON.h" -#include "tglobal.h" #include "hash.h" -#include "dnode.h" -#include "dnodeInt.h" #include "dnodeEps.h" static SDnodeEps *tsEps = NULL; @@ -133,7 +130,7 @@ static void dnodePrintEps(SDnodeEps *eps) { dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); for (int32_t i = 0; i < eps->dnodeNum; i++) { SDnodeEp *ep = &eps->dnodeEps[i]; - dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); + dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); } } @@ -155,7 +152,7 @@ static int32_t dnodeReadEps() { goto PRASE_EPS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PRASE_EPS_OVER; @@ -202,7 +199,7 @@ static int32_t dnodeReadEps() { dError("failed to read %s, dnodeId not found", file); goto PRASE_EPS_OVER; } - ep->dnodeId = dnodeId->valueint; + ep->dnodeId = (int32_t)dnodeId->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { @@ -236,7 +233,14 @@ PRASE_EPS_OVER: dnodeResetEps(eps); if (eps) free(eps); +#if 0 dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort); +#else + if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) { + dError("dnode:%d, localEp is different from %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp); + return -1; + } +#endif terrno = 0; return 0; @@ -273,7 +277,7 @@ static int32_t dnodeWriteEps() { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index 162de2243e71ba7ae4e74839753f98c1cb546237..0dca116d84a8267231acba686018cf6c07ae15e0 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -16,10 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "cJSON.h" -#include "tglobal.h" #include "mnode.h" -#include "dnode.h" -#include "dnodeInt.h" #include "dnodeMInfos.h" static SMInfos tsMInfos; @@ -83,7 +80,7 @@ void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) { pthread_mutex_lock(&tsMInfosMutex); dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); - for (int i = 0; i < ep->numOfEps; ++i) { + for (int32_t i = 0; i < ep->numOfEps; ++i) { ep->port[i] -= TSDB_PORT_DNODEDNODE; dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); } @@ -111,7 +108,7 @@ void dnodeGetMInfos(SMInfos *pMinfos) { void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { pthread_mutex_lock(&tsMInfosMutex); *epSet = tsMEpSet; - for (int i = 0; i < epSet->numOfEps; ++i) { + for (int32_t i = 0; i < epSet->numOfEps; ++i) { epSet->port[i] += TSDB_PORT_DNODEDNODE; } pthread_mutex_unlock(&tsMInfosMutex); @@ -157,12 +154,13 @@ static void dnodeResetMInfos(SMInfos *pMinfos) { } static int32_t dnodeReadMInfos() { - int32_t len = 0; - int32_t maxLen = 2000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - SMInfos minfos = {0}; + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SMInfos minfos = {0}; + bool nodeChanged = false; char file[TSDB_FILENAME_LEN + 20] = {0}; sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); @@ -173,7 +171,7 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_MINFOS_OVER; @@ -191,14 +189,14 @@ static int32_t dnodeReadMInfos() { dError("failed to read mnodeEpSet.json, inUse not found"); goto PARSE_MINFOS_OVER; } - tsMInfos.inUse = inUse->valueint; + tsMInfos.inUse = (int8_t)inUse->valueint; cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { dError("failed to read mnodeEpSet.json, nodeNum not found"); goto PARSE_MINFOS_OVER; } - minfos.mnodeNum = nodeNum->valueint; + minfos.mnodeNum = (int8_t)nodeNum->valueint; cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { @@ -206,13 +204,13 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - int size = cJSON_GetArraySize(nodeInfos); + int32_t size = cJSON_GetArraySize(nodeInfos); if (size != minfos.mnodeNum) { dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); goto PARSE_MINFOS_OVER; } - for (int i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); if (nodeInfo == NULL) continue; @@ -221,14 +219,19 @@ static int32_t dnodeReadMInfos() { dError("failed to read mnodeEpSet.json, nodeId not found"); goto PARSE_MINFOS_OVER; } - minfos.mnodeInfos[i].mnodeId = nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { dError("failed to read mnodeEpSet.json, nodeName not found"); goto PARSE_MINFOS_OVER; } - strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); + + SMInfo *pMinfo = &minfos.mnodeInfos[i]; + pMinfo->mnodeId = (int32_t)nodeId->valueint; + tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); + + bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp); + if (changed) nodeChanged = changed; } dInfo("read file %s successed", file); @@ -245,6 +248,11 @@ PARSE_MINFOS_OVER: dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); } dnodeResetMInfos(&minfos); + + if (nodeChanged) { + dnodeWriteMInfos(); + } + return 0; } @@ -278,7 +286,7 @@ static int32_t dnodeWriteMInfos() { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; @@ -286,3 +294,33 @@ static int32_t dnodeWriteMInfos() { dInfo("successed to write %s", file); return 0; } + +void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { + SRpcConnInfo connInfo = {0}; + rpcGetConnInfo(rpcMsg->handle, &connInfo); + + SRpcEpSet epSet = {0}; + if (forShell) { + dnodeGetEpSetForShell(&epSet); + } else { + dnodeGetEpSetForPeer(&epSet); + } + + dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], + taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse); + + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); + if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) { + if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) || + (epSet.port[i] == tsServerPort && forShell)) { + epSet.inUse = (i + 1) % epSet.numOfEps; + dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse); + } + } + + epSet.port[i] = htons(epSet.port[i]); + } + + rpcSendRedirectRsp(rpcMsg->handle, &epSet); +} \ No newline at end of file diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index ee6dc5212e4fdb98d4bfb7c3a7678fb6bd81bc8c..e4942c49aaba48db8d2a1c74a0af532769de9553 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -15,16 +15,11 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tutil.h" #include "tqueue.h" #include "twal.h" -#include "tglobal.h" #include "mnode.h" -#include "dnode.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" +#include "dnodeVMgmt.h" +#include "dnodeMInfos.h" #include "dnodeMWrite.h" typedef struct { @@ -65,7 +60,7 @@ int32_t dnodeInitMPeer() { void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMPeerQset); } dDebug("dnode mpeer worker:%d is closed", i); @@ -74,7 +69,7 @@ void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; dDebug("dnode mpeer worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mpeer worker:%d join success", i); diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 65f3af7b3bf13efe0ffa80ca5752d66cc4a43e9e..90332e6783bc4861928833d5794f0787f80be993 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -15,16 +15,9 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tutil.h" #include "tqueue.h" -#include "twal.h" -#include "tglobal.h" #include "mnode.h" -#include "dnode.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" +#include "dnodeMInfos.h" #include "dnodeMRead.h" typedef struct { @@ -47,7 +40,7 @@ static void *dnodeProcessMReadQueue(void *param); int32_t dnodeInitMRead() { tsMReadQset = taosOpenQset(); - tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; + tsMReadWP.maxNum = (int32_t)(tsNumOfCores * tsNumOfThreadsPerCore / 2); tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum); tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum); tsMReadWP.curNum = 0; @@ -67,7 +60,7 @@ int32_t dnodeInitMRead() { void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMReadQset); } dDebug("dnode mread worker:%d is closed", i); @@ -76,7 +69,7 @@ void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; dDebug("dnode mread worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mread worker:%d start to join", i); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index ef2d49ef4233eb94f3efd82f77a6918b2b671ffd..79744e153e46777c098644ac9902fb76be81fb11 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -15,17 +15,10 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tutil.h" #include "ttimer.h" #include "tqueue.h" -#include "twal.h" -#include "tglobal.h" #include "mnode.h" -#include "dnode.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" +#include "dnodeMInfos.h" #include "dnodeMWrite.h" typedef struct { @@ -67,7 +60,7 @@ int32_t dnodeInitMWrite() { void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMWriteQset); } dDebug("dnode mwrite worker:%d is closed", i); @@ -76,7 +69,7 @@ void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; dDebug("dnode mwrite worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mwrite worker:%d join success", i); @@ -127,7 +120,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { dnodeSendRedirectMsg(pMsg, true); } else { SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); - dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, + dTrace("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } @@ -136,7 +129,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { } static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { - dDebug("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, + dTrace("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); mnodeCleanupMsg(pWrite); @@ -152,6 +145,14 @@ void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) { return; } + dTrace("msg:%p, app:%p type:%s master:%p will be responsed", pWrite, pWrite->rpcMsg.ahandle, + taosMsg[pWrite->rpcMsg.msgType], pWrite->pBatchMasterMsg); + if (pWrite->pBatchMasterMsg && pWrite != pWrite->pBatchMasterMsg) { + dError("msg:%p, app:%p type:%s master:%p sub message should not response!", pWrite, pWrite->rpcMsg.ahandle, + taosMsg[pWrite->rpcMsg.msgType], pWrite->pBatchMasterMsg); + return; + } + SRpcMsg rpcRsp = { .handle = pWrite->rpcMsg.handle, .pCont = pWrite->rpcRsp.rsp, @@ -191,7 +192,19 @@ void dnodeReprocessMWriteMsg(void *pMsg) { dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); - dnodeSendRedirectMsg(pMsg, true); + if (pWrite->pBatchMasterMsg) { + ++pWrite->pBatchMasterMsg->received; + if (pWrite->pBatchMasterMsg->successed + pWrite->pBatchMasterMsg->received + >= pWrite->pBatchMasterMsg->expected) { + dnodeSendRedirectMsg(&pWrite->pBatchMasterMsg->rpcMsg, true); + dnodeFreeMWriteMsg(pWrite->pBatchMasterMsg); + } + + mnodeDestroySubMsg(pWrite); + + return; + } + dnodeSendRedirectMsg(&pWrite->rpcMsg, true); dnodeFreeMWriteMsg(pWrite); } else { dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 9f52dbd331b02b19348d43309c693f3637765cdd..16f97d0eea8b12bd29d52ed77e9940cf54a5b3ac 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -16,15 +16,14 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taos.h" -#include "tutil.h" +#include "tnote.h" +#include "ttimer.h" #include "tconfig.h" -#include "tglobal.h" #include "tfile.h" #include "twal.h" -#include "trpc.h" -#include "dnode.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" +#include "tfs.h" +#include "tsync.h" +#include "dnodeStep.h" #include "dnodePeer.h" #include "dnodeModule.h" #include "dnodeEps.h" @@ -33,50 +32,47 @@ #include "dnodeCheck.h" #include "dnodeVRead.h" #include "dnodeVWrite.h" +#include "dnodeVMgmt.h" +#include "dnodeVnodes.h" #include "dnodeMRead.h" #include "dnodeMWrite.h" #include "dnodeMPeer.h" #include "dnodeShell.h" #include "dnodeTelemetry.h" +void *tsDnodeTmr = NULL; static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED; static int32_t dnodeInitStorage(); static void dnodeCleanupStorage(); static void dnodeSetRunStatus(SRunStatus status); static void dnodeCheckDataDirOpenned(char *dir); -static int32_t dnodeInitComponents(); -static void dnodeCleanupComponents(int32_t stepId); static int dnodeCreateDir(const char *dir); -typedef struct { - const char *const name; - int32_t (*init)(); - void (*cleanup)(); -} SDnodeComponent; - -static const SDnodeComponent tsDnodeComponents[] = { - {"tfile", tfInit, tfCleanup}, - {"rpc", rpcInit, rpcCleanup}, - {"storage", dnodeInitStorage, dnodeCleanupStorage}, - {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg}, - {"dnodeeps", dnodeInitEps, dnodeCleanupEps}, - {"globalcfg" ,taosCheckGlobalCfg, NULL}, - {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, - {"wal", walInit, walCleanUp}, - {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! - {"vread", dnodeInitVRead, dnodeCleanupVRead}, - {"vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, - {"mread", dnodeInitMRead, dnodeCleanupMRead}, - {"mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, - {"mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, - {"client", dnodeInitClient, dnodeCleanupClient}, - {"server", dnodeInitServer, dnodeCleanupServer}, - {"mgmt", dnodeInitMgmt, dnodeCleanupMgmt}, - {"modules", dnodeInitModules, dnodeCleanupModules}, - {"mgmt-tmr", dnodeInitMgmtTimer, dnodeCleanupMgmtTimer}, - {"shell", dnodeInitShell, dnodeCleanupShell}, - {"telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, +static SStep tsDnodeSteps[] = { + {"dnode-tfile", tfInit, tfCleanup}, + {"dnode-rpc", rpcInit, rpcCleanup}, + {"dnode-globalcfg", taosCheckGlobalCfg, NULL}, + {"dnode-storage", dnodeInitStorage, dnodeCleanupStorage}, + {"dnode-cfg", dnodeInitCfg, dnodeCleanupCfg}, + {"dnode-eps", dnodeInitEps, dnodeCleanupEps}, + {"dnode-minfos", dnodeInitMInfos, dnodeCleanupMInfos}, + {"dnode-wal", walInit, walCleanUp}, + {"dnode-sync", syncInit, syncCleanUp}, + {"dnode-check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! + {"dnode-vread", dnodeInitVRead, dnodeCleanupVRead}, + {"dnode-vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, + {"dnode-vmgmt", dnodeInitVMgmt, dnodeCleanupVMgmt}, + {"dnode-mread", dnodeInitMRead, dnodeCleanupMRead}, + {"dnode-mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, + {"dnode-mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, + {"dnode-client", dnodeInitClient, dnodeCleanupClient}, + {"dnode-server", dnodeInitServer, dnodeCleanupServer}, + {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes}, + {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, + {"dnode-shell", dnodeInitShell, dnodeCleanupShell}, + {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer}, + {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, }; static int dnodeCreateDir(const char *dir) { @@ -87,35 +83,44 @@ static int dnodeCreateDir(const char *dir) { return 0; } -static void dnodeCleanupComponents(int32_t stepId) { - for (int32_t i = stepId; i >= 0; i--) { - if (tsDnodeComponents[i].cleanup) { - (*tsDnodeComponents[i].cleanup)(); - } - } +static void dnodeCleanupComponents() { + int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); + dnodeStepCleanup(tsDnodeSteps, stepSize); } static int32_t dnodeInitComponents() { - int32_t code = 0; - for (int32_t i = 0; i < sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]); i++) { - if (tsDnodeComponents[i].init() != 0) { - dnodeCleanupComponents(i); - code = -1; - break; - } + int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); + return dnodeStepInit(tsDnodeSteps, stepSize); +} + +static int32_t dnodeInitTmr() { + tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); + if (tsDnodeTmr == NULL) { + dError("failed to init dnode timer"); + return -1; + } + + return 0; +} + +static void dnodeCleanupTmr() { + if (tsDnodeTmr != NULL) { + taosTmrCleanUp(tsDnodeTmr); + tsDnodeTmr = NULL; } - return code; } int32_t dnodeInitSystem() { dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); tscEmbedded = 1; + taosIgnSIGPIPE(); taosBlockSIGPIPE(); taosResolveCRC(); taosInitGlobalCfg(); taosReadGlobalLogCfg(); taosSetCoreDump(); - signal(SIGPIPE, SIG_IGN); + taosInitNotes(); + dnodeInitTmr(); if (dnodeCreateDir(tsLogDir) < 0) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); @@ -140,9 +145,9 @@ int32_t dnodeInitSystem() { return -1; } - dnodeStartModules(); dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); + dnodeReportStep("TDengine", "initialized successfully", 1); dInfo("TDengine is initialized successfully"); return 0; @@ -151,7 +156,8 @@ int32_t dnodeInitSystem() { void dnodeCleanUpSystem() { if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) { dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED); - dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1); + dnodeCleanupTmr(); + dnodeCleanupComponents(); taos_cleanup(); taosCloseLog(); } @@ -183,32 +189,35 @@ static void dnodeCheckDataDirOpenned(char *dir) { } static int32_t dnodeInitStorage() { - if (dnodeCreateDir(tsDataDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno)); - return -1; + if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) { + dError("failed to init TFS since %s", tstrerror(terrno)); + return -1; } + strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN); sprintf(tsMnodeDir, "%s/mnode", tsDataDir); sprintf(tsVnodeDir, "%s/vnode", tsDataDir); sprintf(tsDnodeDir, "%s/dnode", tsDataDir); - sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); + // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); return -1; } - //TODO(dengyihao): no need to init here - if (dnodeCreateDir(tsVnodeDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsVnodeDir, strerror(errno)); - return -1; - } + if (dnodeCreateDir(tsDnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno)); return -1; - } - if (dnodeCreateDir(tsVnodeBakDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno)); - return -1; + } + + if (tfsMkdir("vnode") < 0) { + dError("failed to create vnode dir since %s", tstrerror(terrno)); + return -1; + } + + if (tfsMkdir("vnode_bak") < 0) { + dError("failed to create vnode_bak dir since %s", tstrerror(terrno)); + return -1; } dnodeCheckDataDirOpenned(tsDnodeDir); @@ -217,7 +226,7 @@ static int32_t dnodeInitStorage() { return 0; } -static void dnodeCleanupStorage() {} +static void dnodeCleanupStorage() { tfsDestroy(); } bool dnodeIsFirstDeploy() { return strcmp(tsFirst, tsLocalEp) == 0; diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c deleted file mode 100644 index 15378c77c13b3ff423f1fb974c38bf89b5f2e0fa..0000000000000000000000000000000000000000 --- a/src/dnode/src/dnodeMgmt.c +++ /dev/null @@ -1,572 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "cJSON.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "ttimer.h" -#include "tsdb.h" -#include "twal.h" -#include "tqueue.h" -#include "tsync.h" -#include "ttimer.h" -#include "tbn.h" -#include "tglobal.h" -#include "dnode.h" -#include "vnode.h" -#include "mnode.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" -#include "dnodeEps.h" -#include "dnodeCfg.h" -#include "dnodeMInfos.h" -#include "dnodeVRead.h" -#include "dnodeVWrite.h" -#include "dnodeModule.h" - -typedef struct { - pthread_t thread; - int32_t threadIndex; - int32_t failed; - int32_t opened; - int32_t vnodeNum; - int32_t * vnodeList; -} SOpenVnodeThread; - -typedef struct { - SRpcMsg rpcMsg; - char pCont[]; -} SMgmtMsg; - -void * tsDnodeTmr = NULL; -static void * tsStatusTimer = NULL; -static uint32_t tsRebootTime; -static taos_qset tsMgmtQset = NULL; -static taos_queue tsMgmtQueue = NULL; -static pthread_t tsQthread; - -static void dnodeProcessStatusRsp(SRpcMsg *pMsg); -static void dnodeSendStatusMsg(void *handle, void *tmrId); -static void *dnodeProcessMgmtQueue(void *param); - -static int32_t dnodeOpenVnodes(); -static void dnodeCloseVnodes(); -static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg); -static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); - -int32_t dnodeInitMgmt() { - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg; - - dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp); - tsRebootTime = taosGetTimestampSec(); - - int32_t code = vnodeInitResources(); - if (code != TSDB_CODE_SUCCESS) { - dnodeCleanupMgmt(); - return -1; - } - - // create the queue and thread to handle the message - tsMgmtQset = taosOpenQset(); - if (tsMgmtQset == NULL) { - dError("failed to create the mgmt queue set"); - dnodeCleanupMgmt(); - return -1; - } - - tsMgmtQueue = taosOpenQueue(); - if (tsMgmtQueue == NULL) { - dError("failed to create the mgmt queue"); - dnodeCleanupMgmt(); - return -1; - } - - taosAddIntoQset(tsMgmtQset, tsMgmtQueue, NULL); - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - code = pthread_create(&tsQthread, &thAttr, dnodeProcessMgmtQueue, NULL); - pthread_attr_destroy(&thAttr); - if (code != 0) { - dError("failed to create thread to process mgmt queue, reason:%s", strerror(errno)); - dnodeCleanupMgmt(); - return -1; - } - - code = dnodeOpenVnodes(); - if (code != TSDB_CODE_SUCCESS) { - dnodeCleanupMgmt(); - return -1; - } - - dInfo("dnode mgmt is initialized"); - - return TSDB_CODE_SUCCESS; -} - -int32_t dnodeInitMgmtTimer() { - tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); - if (tsDnodeTmr == NULL) { - dError("failed to init dnode timer"); - dnodeCleanupMgmt(); - return -1; - } - - taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer); - dInfo("dnode mgmt timer is initialized"); - return TSDB_CODE_SUCCESS; -} - -void dnodeSendStatusMsgToMnode() { - if (tsDnodeTmr != NULL && tsStatusTimer != NULL) { - dInfo("force send status msg to mnode"); - taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer); - } -} - -void dnodeCleanupMgmtTimer() { - if (tsStatusTimer != NULL) { - taosTmrStopA(&tsStatusTimer); - tsStatusTimer = NULL; - } - - if (tsDnodeTmr != NULL) { - taosTmrCleanUp(tsDnodeTmr); - tsDnodeTmr = NULL; - } -} - -void dnodeCleanupMgmt() { - dnodeCleanupMgmtTimer(); - dnodeCloseVnodes(); - - if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset); - if (tsQthread) pthread_join(tsQthread, NULL); - - if (tsMgmtQueue) taosCloseQueue(tsMgmtQueue); - if (tsMgmtQset) taosCloseQset(tsMgmtQset); - tsMgmtQset = NULL; - tsMgmtQueue = NULL; - - vnodeCleanupResources(); -} - -static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) { - int32_t size = sizeof(SMgmtMsg) + pMsg->contLen; - SMgmtMsg *pMgmt = taosAllocateQitem(size); - if (pMgmt == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - - pMgmt->rpcMsg = *pMsg; - pMgmt->rpcMsg.pCont = pMgmt->pCont; - memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen); - taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt); - - return TSDB_CODE_SUCCESS; -} - -void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) { - int32_t code = dnodeWriteToMgmtQueue(pMsg); - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = code}; - rpcSendResponse(&rsp); - } - - rpcFreeCont(pMsg->pCont); -} - -static void *dnodeProcessMgmtQueue(void *param) { - SMgmtMsg *pMgmt; - SRpcMsg * pMsg; - SRpcMsg rsp = {0}; - int32_t qtype; - void * handle; - - while (1) { - if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) { - dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset); - break; - } - - pMsg = &pMgmt->rpcMsg; - dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]); - if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { - rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); - } else { - rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - } - - rsp.handle = pMsg->handle; - rsp.pCont = NULL; - rpcSendResponse(&rsp); - - taosFreeQitem(pMsg); - } - - return NULL; -} - -static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - DIR *dir = opendir(tsVnodeDir); - if (dir == NULL) { - return TSDB_CODE_DND_NO_WRITE_ACCESS; - } - - *numOfVnodes = 0; - struct dirent *de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; - if (de->d_type & DT_DIR) { - if (strncmp("vnode", de->d_name, 5) != 0) continue; - int32_t vnode = atoi(de->d_name + 5); - if (vnode == 0) continue; - - (*numOfVnodes)++; - - if (*numOfVnodes >= TSDB_MAX_VNODES) { - dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES); - continue; - } else { - vnodeList[*numOfVnodes - 1] = vnode; - } - } - } - closedir(dir); - - return TSDB_CODE_SUCCESS; -} - -static void *dnodeOpenVnode(void *param) { - SOpenVnodeThread *pThread = param; - char vnodeDir[TSDB_FILENAME_LEN * 3]; - - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); - - for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - int32_t vgId = pThread->vnodeList[v]; - snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vgId); - if (vnodeOpen(vgId, vnodeDir) < 0) { - dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); - pThread->failed++; - } else { - dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex); - pThread->opened++; - } - } - - dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); - return NULL; -} - -static int32_t dnodeOpenVnodes() { - int32_t vnodeList[TSDB_MAX_VNODES] = {0}; - int32_t numOfVnodes = 0; - int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); - - if (status != TSDB_CODE_SUCCESS) { - dInfo("get dnode list failed"); - return status; - } - - int32_t threadNum = tsNumOfCores; - int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread)); - for (int32_t t = 0; t < threadNum; ++t) { - threads[t].threadIndex = t; - threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t)); - } - - for (int32_t v = 0; v < numOfVnodes; ++v) { - int32_t t = v % threadNum; - SOpenVnodeThread *pThread = &threads[t]; - pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; - } - - dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes); - - for (int32_t t = 0; t < threadNum; ++t) { - SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum == 0) continue; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - } - - int32_t openVnodes = 0; - int32_t failedVnodes = 0; - for (int32_t t = 0; t < threadNum; ++t) { - SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && pThread->thread) { - pthread_join(pThread->thread, NULL); - } - openVnodes += pThread->opened; - failedVnodes += pThread->failed; - free(pThread->vnodeList); - } - - free(threads); - dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes); - - return TSDB_CODE_SUCCESS; -} - -static void dnodeCloseVnodes() { - int32_t vnodeList[TSDB_MAX_VNODES]= {0}; - int32_t numOfVnodes = 0; - int32_t status; - - status = vnodeGetVnodeList(vnodeList, &numOfVnodes); - - if (status != TSDB_CODE_SUCCESS) { - dInfo("get dnode list failed"); - return; - } - - for (int32_t i = 0; i < numOfVnodes; ++i) { - vnodeClose(vnodeList[i]); - } - - dInfo("total vnodes:%d are all closed", numOfVnodes); -} - -static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = rpcMsg->pCont; - pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); - pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); - pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); - pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); - pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); - pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); - pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); - pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); - pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); - pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); - pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); - pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod); - pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); - - for (int32_t j = 0; j < pCreate->cfg.replications; ++j) { - pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); - } - - return pCreate; -} - -static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg); - - void *pVnode = vnodeAcquire(pCreate->cfg.vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId); - vnodeRelease(pVnode); - return TSDB_CODE_SUCCESS; - } else { - dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId); - return vnodeCreate(pCreate); - } -} - -static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { - SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg); - - void *pVnode = vnodeAcquire(pAlter->cfg.vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId); - int32_t code = vnodeAlter(pVnode, pAlter); - vnodeRelease(pVnode); - return code; - } else { - dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId); - return TSDB_CODE_VND_INVALID_VGROUP_ID; - } -} - -static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = rpcMsg->pCont; - pDrop->vgId = htonl(pDrop->vgId); - - return vnodeDrop(pDrop->vgId); -} - -static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) { -// SAlterStreamMsg *pStream = pCont; -// pStream->uid = htobe64(pStream->uid); -// pStream->stime = htobe64(pStream->stime); -// pStream->vnode = htonl(pStream->vnode); -// pStream->sid = htonl(pStream->sid); -// pStream->status = htonl(pStream->status); -// -// int32_t code = dnodeCreateStream(pStream); - - return 0; -} - -static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { - SCfgDnodeMsg *pCfg = pMsg->pCont; - return taosCfgDynamicOptions(pCfg->config); -} - -static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { - SCreateMnodeMsg *pCfg = pMsg->pCont; - pCfg->dnodeId = htonl(pCfg->dnodeId); - if (pCfg->dnodeId != dnodeGetDnodeId()) { - dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); - return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; - } - - if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) { - dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp); - return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; - } - - dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum); - for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) { - pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId); - dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); - } - - dnodeStartMnode(&pCfg->mnodes); - - return TSDB_CODE_SUCCESS; -} - -static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { - if (pMsg->code != TSDB_CODE_SUCCESS) { - dError("status rsp is received, error:%s", tstrerror(pMsg->code)); - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - return; - } - - SStatusRsp *pStatusRsp = pMsg->pCont; - SMInfos *pMinfos = &pStatusRsp->mnodes; - dnodeUpdateMInfos(pMinfos); - - SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; - pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); - pCfg->moduleStatus = htonl(pCfg->moduleStatus); - pCfg->dnodeId = htonl(pCfg->dnodeId); - dnodeUpdateCfg(pCfg); - - vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes); - - SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); - dnodeUpdateEps(pEps); - - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); -} - -static void dnodeSendStatusMsg(void *handle, void *tmrId) { - if (tsDnodeTmr == NULL) { - dError("dnode timer is already released"); - return; - } - - if (tsStatusTimer == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - dError("failed to start status timer"); - return; - } - - int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); - SStatusMsg *pStatus = rpcMallocCont(contLen); - if (pStatus == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - dError("failed to malloc status message"); - return; - } - - dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId); - pStatus->dnodeId = htonl(dnodeGetDnodeId()); - pStatus->version = htonl(tsVersion); - pStatus->lastReboot = htonl(tsRebootTime); - pStatus->numOfCores = htons((uint16_t) tsNumOfCores); - pStatus->diskAvailable = tsAvailDataDirGB; - pStatus->alternativeRole = (uint8_t) tsAlternativeRole; - tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); - - // fill cluster cfg parameters - pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes); - pStatus->clusterCfg.enableBalance = htonl(tsEnableBalance); - pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum); - pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold); - pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); - pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode); - pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); - tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN); - tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); - pStatus->clusterCfg.checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); - - vnodeBuildStatusMsg(pStatus); - contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); - pStatus->openVnodes = htons(pStatus->openVnodes); - - SRpcMsg rpcMsg = { - .pCont = pStatus, - .contLen = contLen, - .msgType = TSDB_MSG_TYPE_DM_STATUS - }; - - SRpcEpSet epSet; - dnodeGetEpSetForPeer(&epSet); - dnodeSendMsgToDnode(&epSet, &rpcMsg); -} - -void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { - SRpcConnInfo connInfo = {0}; - rpcGetConnInfo(rpcMsg->handle, &connInfo); - - SRpcEpSet epSet = {0}; - if (forShell) { - dnodeGetEpSetForShell(&epSet); - } else { - dnodeGetEpSetForPeer(&epSet); - } - - dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], - taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse); - - for (int i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); - epSet.port[i] = htons(epSet.port[i]); - } - - rpcSendRedirectRsp(rpcMsg->handle, &epSet); -} diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index f664618f516c2cb36d116e3e4f47ed6daf28e042..a661585b3b39df986ac7866a255472e47e789fe6 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -15,15 +15,10 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" -#include "taosmsg.h" -#include "tglobal.h" #include "mnode.h" #include "http.h" #include "tmqtt.h" #include "monitor.h" -#include "dnode.h" -#include "dnodeInt.h" #include "dnodeModule.h" typedef struct { @@ -102,6 +97,20 @@ void dnodeCleanupModules() { } } +static int32_t dnodeStartModules() { + for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { + if (tsModule[module].enable && tsModule[module].startFp) { + int32_t code = (*tsModule[module].startFp)(); + if (code != 0) { + dError("failed to start module:%s, code:%d", tsModule[module].name, code); + return code; + } + } + } + + return 0; +} + int32_t dnodeInitModules() { dnodeAllocModules(); @@ -115,27 +124,19 @@ int32_t dnodeInitModules() { } dInfo("dnode modules is initialized"); - return 0; + return dnodeStartModules(); } -void dnodeStartModules() { - for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { - if (tsModule[module].enable && tsModule[module].startFp) { - if ((*tsModule[module].startFp)() != 0) { - dError("failed to start module:%s", tsModule[module].name); - } - } - } -} +int32_t dnodeProcessModuleStatus(uint32_t moduleStatus) { + int32_t code = 0; -void dnodeProcessModuleStatus(uint32_t moduleStatus) { for (int32_t module = TSDB_MOD_MNODE; module < TSDB_MOD_HTTP; ++module) { bool enableModule = moduleStatus & (1 << module); if (!tsModule[module].enable && enableModule) { dInfo("module status:%u is set, start %s module", moduleStatus, tsModule[module].name); tsModule[module].enable = true; dnodeSetModuleStatus(module); - (*tsModule[module].startFp)(); + code = (*tsModule[module].startFp)(); } if (tsModule[module].enable && !enableModule) { @@ -145,21 +146,29 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) { (*tsModule[module].stopFp)(); } } -} -bool dnodeStartMnode(SMInfos *pMinfos) { - SMInfos *pMnodes = pMinfos; + return code; +} +int32_t dnodeStartMnode(SMInfos *pMinfos) { if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) { dDebug("mnode module is already started, module status:%d", tsModuleStatus); - return false; + return 0; } uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE); dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus); - dnodeProcessModuleStatus(moduleStatus); - sdbUpdateSync(pMnodes); + int32_t code = dnodeProcessModuleStatus(moduleStatus); + if (code == 0) { + code = sdbUpdateSync(pMinfos); + } + + if (code != 0) { + dError("failed to start mnode module since %s", tstrerror(code)); + moduleStatus = tsModuleStatus & ~(1 << TSDB_MOD_MNODE); + dnodeProcessModuleStatus(moduleStatus); + } - return true; + return code; } diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 6b5b28622b70f120132ecb8c1f90ef2d56130b5c..79c60874f99cca535955246f9fc393b1a7cee638 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -21,15 +21,12 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosmsg.h" -#include "tglobal.h" #include "mnode.h" -#include "dnode.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" +#include "dnodeVMgmt.h" #include "dnodeVWrite.h" #include "dnodeMPeer.h" #include "dnodeMInfos.h" +#include "dnodeStep.h" static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); @@ -44,26 +41,26 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToVMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToVMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMPeerQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue; - + SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = tsDnodeDnodePort; rpcInit.label = "DND-S"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessReqMsgFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES; + rpcInit.sessions = TSDB_MAX_VNODES << 4; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; @@ -91,14 +88,18 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { .pCont = NULL, .contLen = 0 }; - + if (pMsg->pCont == NULL) return; + if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + dnodeSendStartupStep(pMsg); + return; + } if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { rspMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); - dDebug("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); + dTrace("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); return; } @@ -125,7 +126,7 @@ int32_t dnodeInitClient() { rpcInit.label = "DND-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessRspFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES; + rpcInit.sessions = TSDB_MAX_VNODES << 4; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "t"; @@ -153,7 +154,7 @@ void dnodeCleanupClient() { static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; - dDebug("msg:%p is ignored since dnode is stopping", pMsg); + dTrace("msg:%p is ignored since dnode is stopping", pMsg); rpcFreeCont(pMsg->pCont); return; } diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 89f657f78986b8e57c0b5e1dedb841c451db00ba..9a226b81e8eddc13aa5cb37ca422b68bcb808e03 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -15,20 +15,14 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "taosdef.h" -#include "taosmsg.h" -#include "tglobal.h" -#include "tutil.h" #include "http.h" #include "mnode.h" -#include "dnode.h" -#include "dnodeInt.h" #include "dnodeVRead.h" #include "dnodeVWrite.h" #include "dnodeMRead.h" #include "dnodeMWrite.h" #include "dnodeShell.h" +#include "dnodeStep.h" static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *); @@ -42,12 +36,12 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; - + // the following message shall be treated as mnode write dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue; @@ -63,19 +57,20 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue; - - // the following message shall be treated as mnode query + + // the following message shall be treated as mnode query dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue; - int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore; - numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0); + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep; + + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } @@ -142,7 +137,23 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } } +static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (strcmp(user, "nettestinternal") == 0) { + char pass[32] = {0}; + taosEncryptPass((uint8_t *)user, strlen(user), pass); + *spi = 0; + *encrypt = 0; + *ckey = 0; + memcpy(secret, pass, TSDB_KEY_LEN); + dTrace("nettest user is authorized"); + return 0; + } + + return -1; +} + static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0; int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); if (code != TSDB_CODE_APP_NOT_READY) return code; @@ -153,7 +164,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char rpcMsg.pCont = pMsg; rpcMsg.contLen = sizeof(SAuthMsg); rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; - + dDebug("user:%s, send auth msg to mnodes", user); SRpcMsg rpcRsp = {0}; dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); @@ -191,21 +202,21 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { SRpcMsg rpcRsp = {0}; dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); terrno = rpcRsp.code; - + if (rpcRsp.code != 0) { rpcFreeCont(rpcRsp.pCont); dError("vgId:%d, tid:%d failed to config table from mnode", vgId, tid); return NULL; } else { dInfo("vgId:%d, tid:%d config table msg is received", vgId, tid); - + // delete this after debug finished SMDCreateTableMsg *pTable = rpcRsp.pCont; int16_t numOfColumns = htons(pTable->numOfColumns); int16_t numOfTags = htons(pTable->numOfTags); int32_t tableId = htonl(pTable->tid); uint64_t uid = htobe64(pTable->uid); - dInfo("table:%s, numOfColumns:%d numOfTags:%d tid:%d uid:%" PRIu64, pTable->tableId, numOfColumns, numOfTags, tableId, uid); + dInfo("table:%s, numOfColumns:%d numOfTags:%d tid:%d uid:%" PRIu64, pTable->tableFname, numOfColumns, numOfTags, tableId, uid); return rpcRsp.pCont; } diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c new file mode 100644 index 0000000000000000000000000000000000000000..eda6fb227dd6bf29499933383bb82f83ec851ee4 --- /dev/null +++ b/src/dnode/src/dnodeStep.c @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "dnodeStep.h" + +static SStartupStep tsStartupStep; + +void dnodeReportStep(char *name, char *desc, int8_t finished) { + tstrncpy(tsStartupStep.name, name, sizeof(tsStartupStep.name)); + tstrncpy(tsStartupStep.desc, desc, sizeof(tsStartupStep.desc)); + tsStartupStep.finished = finished; +} + +void dnodeSendStartupStep(SRpcMsg *pMsg) { + dInfo("nettest msg is received, cont:%s", (char *)pMsg->pCont); + + SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep)); + memcpy(pStep, &tsStartupStep, sizeof(SStartupStep)); + + dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished); + + SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStep, .contLen = sizeof(SStartupStep)}; + rpcSendResponse(&rpcRsp); + rpcFreeCont(pMsg->pCont); +} + +void taosStepCleanupImp(SStep *pSteps, int32_t stepId) { + for (int32_t step = stepId; step >= 0; step--) { + SStep *pStep = pSteps + step; + dDebug("step:%s will cleanup", pStep->name); + if (pStep->cleanupFp != NULL) { + (*pStep->cleanupFp)(); + } + } +} + +int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) { + for (int32_t step = 0; step < stepSize; step++) { + SStep *pStep = pSteps + step; + if (pStep->initFp == NULL) continue; + + dnodeReportStep(pStep->name, "Start initialization", 0); + + int32_t code = (*pStep->initFp)(); + if (code != 0) { + dDebug("step:%s will cleanup", pStep->name); + taosStepCleanupImp(pSteps, step); + return code; + } + dInfo("step:%s is initialized", pStep->name); + + dnodeReportStep(pStep->name, "Initialization complete", 0); + } + + return 0; +} + +void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { + taosStepCleanupImp(pSteps, stepSize - 1); +} diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 56316e9619344243ab2e5fa46e0acfc6e0331e8a..e49b3eba99408c9453189106dc6f01e0d0afc7fe 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -16,19 +16,21 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tgrant.h" -#include "tutil.h" -#include "tglobal.h" -#include "dnodeInt.h" +#include "tconfig.h" #include "dnodeMain.h" -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; +static void siguser1Handler(int32_t signum, void *sigInfo, void *context); +static void siguser2Handler(int32_t signum, void *sigInfo, void *context); +static void sigintHandler(int32_t signum, void *sigInfo, void *context); int32_t main(int32_t argc, char *argv[]) { + int dump_config = 0; + // Set global configuration file for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { + if (i < argc - 1) { if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { printf("config file path overflow"); exit(EXIT_FAILURE); @@ -38,6 +40,8 @@ int32_t main(int32_t argc, char *argv[]) { printf("'-c' requires a parameter, default:%s\n", configDir); exit(EXIT_FAILURE); } + } else if (strcmp(argv[i], "-C") == 0) { + dump_config = 1; } else if (strcmp(argv[i], "-V") == 0) { #ifdef _ACCT char *versionStr = "enterprise"; @@ -78,8 +82,8 @@ int32_t main(int32_t argc, char *argv[]) { taosSetRandomFileFailOutput(NULL); } } else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { - if ( (i+1) < argc ) { - int factor = atoi(argv[i+1]); + if ((i + 1) < argc) { + int factor = atoi(argv[i + 1]); printf("The factor of random failure is %d\n", factor); taosSetRandomFileFailFactor(factor); } else { @@ -90,20 +94,33 @@ int32_t main(int32_t argc, char *argv[]) { #endif } + if (0 != dump_config) { + tscEmbedded = 1; + taosInitGlobalCfg(); + taosReadGlobalLogCfg(); + + if (!taosReadGlobalCfg()) { + printf("TDengine read global config failed"); + exit(EXIT_FAILURE); + } + + taosDumpGlobalCfg(); + exit(EXIT_SUCCESS); + } + if (tsem_init(&exitSem, 0, 0) != 0) { printf("failed to create exit semphore\n"); exit(EXIT_FAILURE); } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = signal_handler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); + taosSetSignal(SIGUSR1, siguser1Handler); + taosSetSignal(SIGUSR2, siguser2Handler); + taosSetSignal(SIGTERM, sigintHandler); + taosSetSignal(SIGHUP, sigintHandler); + taosSetSignal(SIGINT, sigintHandler); + taosSetSignal(SIGABRT, sigintHandler); + taosSetSignal(SIGBREAK, sigintHandler); // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -128,33 +145,40 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Shut down TDengine service successfully"); dInfo("TDengine is shut down!"); closelog(); + +#ifdef WINDOWS + tsem_post(&exitSem); +#endif return EXIT_SUCCESS; } -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } +static void siguser1Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("debugFlag 143"); } - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // clean the system. - dInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); +static void siguser2Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("resetlog"); } +static void sigintHandler(int32_t signum, void *sigInfo, void *context) { // protect the application from receive another signal - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); + taosIgnSignal(SIGUSR1); + taosIgnSignal(SIGUSR2); + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGBREAK); + + // clean the system. + dInfo("shut down signal is %d", signum); + +#ifndef WINDOWS + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); +#endif + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); // inform main thread to exit tsem_post(&exitSem); +#ifdef WINDOWS + tsem_wait(&exitSem); +#endif } diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index e973f9901f19b7aa6f4d4958d88e01ba258a82bb..4caece16612353155886ff20055433878ec7411c 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -15,9 +15,6 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tutil.h" #include "osTime.h" #include "tsocket.h" #include "tbuffer.h" @@ -32,16 +29,18 @@ #include "mnodeTable.h" #include "mnodeSdb.h" #include "mnodeAcct.h" -#include "dnode.h" -#include "dnodeInt.h" #include "dnodeTelemetry.h" -static tsem_t tsExitSem; +// sem_timedwait is NOT implemented on MacOSX +// thus, we use pthread_mutex_t/pthread_cond_t to simulate +static pthread_mutex_t tsExitLock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t tsExitCond = PTHREAD_COND_INITIALIZER; +static volatile int tsExit = 0; static pthread_t tsTelemetryThread; #define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +#define REPORT_INTERVAL 86400 static void beginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); @@ -98,14 +97,14 @@ static void addStringField(SBufferWriter* bw, const char* k, const char* v) { static void addCpuInfo(SBufferWriter* bw) { char * line = NULL; size_t size = 0; - int done = 0; + int32_t done = 0; FILE* fp = fopen("/proc/cpuinfo", "r"); if (fp == NULL) { return; } - while (done != 3 && (size = getline(&line, &size, fp)) != -1) { + while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (((done&1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -134,7 +133,7 @@ static void addOsInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -160,7 +159,7 @@ static void addMemoryInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -177,8 +176,8 @@ static void addMemoryInfo(SBufferWriter* bw) { static void addVersionInfo(SBufferWriter* bw) { addStringField(bw, "version", version); addStringField(bw, "buildInfo", buildinfo); - addStringField(bw, "gitInfo", gitinfo); - addStringField(bw, "email", tsEmail); + addStringField(bw, "gitInfo", gitinfo); + addStringField(bw, "email", tsEmail); } static void addRuntimeInfo(SBufferWriter* bw) { @@ -200,12 +199,12 @@ static void addRuntimeInfo(SBufferWriter* bw) { static void sendTelemetryReport() { char buf[128]; - uint32_t ip = taosGetIpFromFqdn(TELEMETRY_SERVER); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - int fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; @@ -227,10 +226,10 @@ static void sendTelemetryReport() { "Content-Type: application/json\n" "Content-Length: "; - taosWriteSocket(fd, header, strlen(header)); - int contLen = tbufTell(&bw) - 1; + taosWriteSocket(fd, header, (int32_t)strlen(header)); + int32_t contLen = (int32_t)(tbufTell(&bw) - 1); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, strlen(buf)); + taosWriteSocket(fd, buf, (int32_t)strlen(buf)); taosWriteSocket(fd, tbufGetData(&bw, false), contLen); tbufCloseWriter(&bw); @@ -246,12 +245,14 @@ static void* telemetryThread(void* param) { clock_gettime(CLOCK_REALTIME, &end); end.tv_sec += 300; // wait 5 minutes before send first report - while (1) { - if (sem_timedwait(&tsExitSem, &end) == 0) { - break; - } else if (errno != ETIMEDOUT) { - continue; - } + while (!tsExit) { + int r = 0; + struct timespec ts = end; + pthread_mutex_lock(&tsExitLock); + r = pthread_cond_timedwait(&tsExitCond, &tsExitLock, &ts); + pthread_mutex_unlock(&tsExitLock); + if (r==0) break; + if (r!=ETIMEDOUT) continue; if (sdbIsMaster()) { sendTelemetryReport(); @@ -263,31 +264,24 @@ static void* telemetryThread(void* param) { } static void dnodeGetEmail(char* filepath) { - int fd = open(filepath, O_RDONLY); + int32_t fd = open(filepath, O_RDONLY); if (fd < 0) { return; } - + if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); - } + } - close(fd); + taosClose(fd); } - int32_t dnodeInitTelemetry() { if (!tsEnableTelemetryReporting) { return 0; } - dnodeGetEmail("/usr/local/taos/email"); - - if (tsem_init(&tsExitSem, 0, 0) == -1) { - // just log the error, it is ok for telemetry to fail - dTrace("failed to create semaphore for telemetry, reason:%s", strerror(errno)); - return 0; - } + dnodeGetEmail("/usr/local/taos/email"); pthread_attr_t attr; pthread_attr_init(&attr); @@ -296,7 +290,7 @@ int32_t dnodeInitTelemetry() { int32_t code = pthread_create(&tsTelemetryThread, &attr, telemetryThread, NULL); pthread_attr_destroy(&attr); if (code != 0) { - dTrace("failed to create telemetry thread, reason:%s", strerror(errno)); + dTrace("failed to create telemetry thread, reason:%s", strerror(code)); } dInfo("dnode telemetry is initialized"); @@ -308,9 +302,12 @@ void dnodeCleanupTelemetry() { return; } - if (tsTelemetryThread) { - tsem_post(&tsExitSem); + if (taosCheckPthreadValid(tsTelemetryThread)) { + pthread_mutex_lock(&tsExitLock); + tsExit = 1; + pthread_cond_signal(&tsExitCond); + pthread_mutex_unlock(&tsExitLock); + pthread_join(tsTelemetryThread, NULL); - tsem_destroy(&tsExitSem); } } diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c new file mode 100644 index 0000000000000000000000000000000000000000..1e428fc8b13e2a476868738f043c90914c61f5fc --- /dev/null +++ b/src/dnode/src/dnodeVMgmt.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tqueue.h" +#include "tworker.h" +#include "dnodeVMgmt.h" + +typedef struct { + SRpcMsg rpcMsg; + char pCont[]; +} SMgmtMsg; + +static SWorkerPool tsVMgmtWP; +static taos_queue tsVMgmtQueue = NULL; + +static void * dnodeProcessMgmtQueue(void *param); +static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg); +static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); + +int32_t dnodeInitVMgmt() { + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg; + + int32_t code = vnodeInitMgmt(); + if (code != TSDB_CODE_SUCCESS) return -1; + + tsVMgmtWP.name = "vmgmt"; + tsVMgmtWP.workerFp = dnodeProcessMgmtQueue; + tsVMgmtWP.min = 1; + tsVMgmtWP.max = 1; + if (tWorkerInit(&tsVMgmtWP) != 0) return -1; + + tsVMgmtQueue = tWorkerAllocQueue(&tsVMgmtWP, NULL); + + dInfo("dnode vmgmt is initialized"); + return TSDB_CODE_SUCCESS; +} + +void dnodeCleanupVMgmt() { + tWorkerFreeQueue(&tsVMgmtWP, tsVMgmtQueue); + tWorkerCleanup(&tsVMgmtWP); + + tsVMgmtQueue = NULL; + vnodeCleanupMgmt(); +} + +static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) { + int32_t size = sizeof(SMgmtMsg) + pMsg->contLen; + SMgmtMsg *pMgmt = taosAllocateQitem(size); + if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; + + pMgmt->rpcMsg = *pMsg; + pMgmt->rpcMsg.pCont = pMgmt->pCont; + memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen); + taosWriteQitem(tsVMgmtQueue, TAOS_QTYPE_RPC, pMgmt); + + return TSDB_CODE_SUCCESS; +} + +void dnodeDispatchToVMgmtQueue(SRpcMsg *pMsg) { + int32_t code = dnodeWriteToMgmtQueue(pMsg); + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = code}; + rpcSendResponse(&rsp); + } + + rpcFreeCont(pMsg->pCont); +} + +static void *dnodeProcessMgmtQueue(void *wparam) { + SWorker * pWorker = wparam; + SWorkerPool *pPool = pWorker->pPool; + SMgmtMsg * pMgmt; + SRpcMsg * pMsg; + SRpcMsg rsp = {0}; + int32_t qtype; + void * handle; + + while (1) { + if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pMgmt, &handle) == 0) { + dDebug("qdnode mgmt got no message from qset:%p, , exit", pPool->qset); + break; + } + + pMsg = &pMgmt->rpcMsg; + dTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]); + if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { + rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); + } else { + rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; + } + + dTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code); + if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) { + rsp.handle = pMsg->handle; + rsp.pCont = NULL; + rpcSendResponse(&rsp); + } + + taosFreeQitem(pMsg); + } + + return NULL; +} + +static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { + SCreateVnodeMsg *pCreate = rpcMsg->pCont; + pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); + pCreate->cfg.dbCfgVersion = htonl(pCreate->cfg.dbCfgVersion); + pCreate->cfg.vgCfgVersion = htonl(pCreate->cfg.vgCfgVersion); + pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); + pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); + pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); + pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); + pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); + pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); + pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); + pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); + pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); + pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod); + pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); + + for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) { + pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); + } + + return pCreate; +} + +static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { + SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg); + + void *pVnode = vnodeAcquire(pCreate->cfg.vgId); + if (pVnode != NULL) { + dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId); + vnodeRelease(pVnode); + return TSDB_CODE_SUCCESS; + } else { + dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId); + return vnodeCreate(pCreate); + } +} + +static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { + SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg); + + void *pVnode = vnodeAcquire(pAlter->cfg.vgId); + if (pVnode != NULL) { + dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId); + int32_t code = vnodeAlter(pVnode, pAlter); + vnodeRelease(pVnode); + return code; + } else { + dInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } +} + +static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { + SDropVnodeMsg *pDrop = rpcMsg->pCont; + pDrop->vgId = htonl(pDrop->vgId); + + return vnodeDrop(pDrop->vgId); +} + +static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) { + return 0; +} + +static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { + SCfgDnodeMsg *pCfg = pMsg->pCont; + return taosCfgDynamicOptions(pCfg->config); +} + +static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { + SCreateMnodeMsg *pCfg = pMsg->pCont; + pCfg->dnodeId = htonl(pCfg->dnodeId); + if (pCfg->dnodeId != dnodeGetDnodeId()) { + dDebug("dnode:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); + return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; + } + + if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) { + dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp); + return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; + } + + dDebug("dnode:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum); + for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) { + pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId); + dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); + } + + return dnodeStartMnode(&pCfg->mnodes); +} diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index b42a627a3a78031be3191d7be5fb1202723dcc57..ea738661ce2813e13468ad91b4dc1d54775db21f 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -15,76 +15,46 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tglobal.h" #include "tqueue.h" -#include "vnode.h" -#include "dnodeInt.h" - -typedef struct { - pthread_t thread; // thread - int32_t workerId; // worker ID -} SVReadWorker; - -typedef struct { - int32_t max; // max number of workers - int32_t min; // min number of workers - int32_t num; // current number of workers - SVReadWorker * worker; - pthread_mutex_t mutex; -} SVReadWorkerPool; +#include "tworker.h" +#include "dnodeVRead.h" static void *dnodeProcessReadQueue(void *pWorker); // module global variable -static SVReadWorkerPool tsVReadWP; -static taos_qset tsVReadQset; +static SWorkerPool tsVQueryWP; +static SWorkerPool tsVFetchWP; int32_t dnodeInitVRead() { - tsVReadQset = taosOpenQset(); - - tsVReadWP.min = tsNumOfCores; - tsVReadWP.max = tsNumOfCores * tsNumOfThreadsPerCore; - if (tsVReadWP.max <= tsVReadWP.min * 2) tsVReadWP.max = 2 * tsVReadWP.min; - tsVReadWP.worker = calloc(sizeof(SVReadWorker), tsVReadWP.max); - pthread_mutex_init(&tsVReadWP.mutex, NULL); - - if (tsVReadWP.worker == NULL) return -1; - for (int i = 0; i < tsVReadWP.max; ++i) { - SVReadWorker *pWorker = tsVReadWP.worker + i; - pWorker->workerId = i; - } + const int32_t maxFetchThreads = 4; - dInfo("dnode vread is initialized, min worker:%d max worker:%d", tsVReadWP.min, tsVReadWP.max); - return 0; -} + // calculate the available query thread + float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1); -void dnodeCleanupVRead() { - for (int i = 0; i < tsVReadWP.max; ++i) { - SVReadWorker *pWorker = tsVReadWP.worker + i; - if (pWorker->thread) { - taosQsetThreadResume(tsVReadQset); - } - } + tsVQueryWP.name = "vquery"; + tsVQueryWP.workerFp = dnodeProcessReadQueue; + tsVQueryWP.min = (int32_t) threadsForQuery; + tsVQueryWP.max = tsVQueryWP.min; + if (tWorkerInit(&tsVQueryWP) != 0) return -1; - for (int i = 0; i < tsVReadWP.max; ++i) { - SVReadWorker *pWorker = tsVReadWP.worker + i; - if (pWorker->thread) { - pthread_join(pWorker->thread, NULL); - } - } + tsVFetchWP.name = "vfetch"; + tsVFetchWP.workerFp = dnodeProcessReadQueue; + tsVFetchWP.min = MIN(maxFetchThreads, tsNumOfCores); + tsVFetchWP.max = tsVFetchWP.min; + if (tWorkerInit(&tsVFetchWP) != 0) return -1; - free(tsVReadWP.worker); - taosCloseQset(tsVReadQset); - pthread_mutex_destroy(&tsVReadWP.mutex); + return 0; +} - dInfo("dnode vread is closed"); +void dnodeCleanupVRead() { + tWorkerCleanup(&tsVFetchWP); + tWorkerCleanup(&tsVQueryWP); } void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { int32_t queuedMsgNum = 0; int32_t leftLen = pMsg->contLen; + int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID; char * pCont = pMsg->pCont; while (leftLen > 0) { @@ -92,9 +62,10 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { pHead->vgId = htonl(pHead->vgId); pHead->contLen = htonl(pHead->contLen); + assert(pHead->contLen > 0); void *pVnode = vnodeAcquire(pHead->vgId); if (pVnode != NULL) { - int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; vnodeRelease(pVnode); } @@ -104,50 +75,27 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { } if (queuedMsgNum == 0) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code}; rpcSendResponse(&rpcRsp); } rpcFreeCont(pMsg->pCont); } -void *dnodeAllocVReadQueue(void *pVnode) { - pthread_mutex_lock(&tsVReadWP.mutex); - taos_queue queue = taosOpenQueue(); - if (queue == NULL) { - pthread_mutex_unlock(&tsVReadWP.mutex); - return NULL; - } - - taosAddIntoQset(tsVReadQset, queue, pVnode); - - // spawn a thread to process queue - if (tsVReadWP.num < tsVReadWP.max) { - do { - SVReadWorker *pWorker = tsVReadWP.worker + tsVReadWP.num; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) { - dError("failed to create thread to process vread vqueue since %s", strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - tsVReadWP.num++; - dDebug("dnode vread worker:%d is launched, total:%d", pWorker->workerId, tsVReadWP.num); - } while (tsVReadWP.num < tsVReadWP.min); - } +void *dnodeAllocVQueryQueue(void *pVnode) { + return tWorkerAllocQueue(&tsVQueryWP, pVnode); +} - pthread_mutex_unlock(&tsVReadWP.mutex); - dDebug("pVnode:%p, dnode vread queue:%p is allocated", pVnode, queue); +void *dnodeAllocVFetchQueue(void *pVnode) { + return tWorkerAllocQueue(&tsVFetchWP, pVnode); +} - return queue; +void dnodeFreeVQueryQueue(void *pQqueue) { + tWorkerFreeQueue(&tsVQueryWP, pQqueue); } -void dnodeFreeVReadQueue(void *pRqueue) { - taosCloseQueue(pRqueue); +void dnodeFreeVFetchQueue(void *pFqueue) { + tWorkerFreeQueue(&tsVFetchWP, pFqueue); } void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { @@ -164,18 +112,20 @@ void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { } -static void *dnodeProcessReadQueue(void *pWorker) { - SVReadMsg *pRead; - int32_t qtype; - void * pVnode; +static void *dnodeProcessReadQueue(void *wparam) { + SWorker * pWorker = wparam; + SWorkerPool *pPool = pWorker->pPool; + SVReadMsg * pRead; + int32_t qtype; + void * pVnode; while (1) { - if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) { - dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset); + if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pRead, &pVnode) == 0) { + dDebug("dnode vquery got no message from qset:%p, exiting", pPool->qset); break; } - dTrace("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle, + dTrace("msg:%p, app:%p type:%s will be processed in vquery queue, qtype:%d", pRead, pRead->rpcAhandle, taosMsg[pRead->msgType], qtype); int32_t code = vnodeProcessRead(pVnode, pRead); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 6d4b50ee542b565669f0a4782142a3919c19ebe8..a3ff459396a974eae26be3561e418b5836b90f4b 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -15,13 +15,8 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tglobal.h" #include "tqueue.h" -#include "twal.h" -#include "vnode.h" -#include "dnodeInt.h" +#include "dnodeVWrite.h" typedef struct { taos_qall qall; @@ -57,14 +52,14 @@ int32_t dnodeInitVWrite() { void dnodeCleanupVWrite() { for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { - taosQsetThreadResume(pWorker->qset); + if (taosCheckPthreadValid(pWorker->thread)) { + if (pWorker->qset) taosQsetThreadResume(pWorker->qset); } } for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); @@ -193,6 +188,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { int32_t numOfMsgs; int32_t qtype; + taosBlockSIGPIPE(); dDebug("dnode vwrite worker:%d is running", pWorker->workerId); while (1) { @@ -206,13 +202,14 @@ static void *dnodeProcessVWriteQueue(void *wparam) { for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite, - pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); + pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead.msgType], qtypeStr[qtype], pWrite->pHead.version); - pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); + pWrite->code = vnodeProcessWrite(pVnode, &pWrite->pHead, qtype, pWrite); if (pWrite->code <= 0) pWrite->processedCount = 1; - if (pWrite->code == 0 && pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; + if (pWrite->code > 0) pWrite->code = 0; + if (pWrite->code == 0 && pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; - dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code)); + dTrace("msg:%p is processed in vwrite queue, code:0x%x", pWrite, pWrite->code); } walFsync(vnodeGetWal(pVnode), forceFsync); @@ -225,7 +222,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code); } else { if (qtype == TAOS_QTYPE_FWD) { - vnodeConfirmForward(pVnode, pWrite->pHead->version, 0); + vnodeConfirmForward(pVnode, pWrite->pHead.version, 0); } if (pWrite->rspRet.rsp) { rpcFreeCont(pWrite->rspRet.rsp); diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c new file mode 100644 index 0000000000000000000000000000000000000000..9f32541612c48d9d68cbbbb0799a55ca2c2e5838 --- /dev/null +++ b/src/dnode/src/dnodeVnodes.c @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "ttimer.h" +#include "dnodeEps.h" +#include "dnodeCfg.h" +#include "dnodeMInfos.h" +#include "dnodeVnodes.h" + +typedef struct { + pthread_t thread; + int32_t threadIndex; + int32_t failed; + int32_t opened; + int32_t vnodeNum; + int32_t * vnodeList; +} SOpenVnodeThread; + +extern void * tsDnodeTmr; +static void * tsStatusTimer = NULL; +static uint32_t tsRebootTime = 0; +static int32_t tsOpenVnodes = 0; +static int32_t tsTotalVnodes = 0; + +static void dnodeSendStatusMsg(void *handle, void *tmrId); +static void dnodeProcessStatusRsp(SRpcMsg *pMsg); + +int32_t dnodeInitStatusTimer() { + dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp); + + tsRebootTime = taosGetTimestampSec(); + taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer); + + dInfo("dnode status timer is initialized"); + return TSDB_CODE_SUCCESS; +} + +void dnodeCleanupStatusTimer() { + if (tsStatusTimer != NULL) { + taosTmrStopA(&tsStatusTimer); + tsStatusTimer = NULL; + } +} + +static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { + DIR *dir = opendir(tsVnodeDir); + if (dir == NULL) return TSDB_CODE_DND_NO_WRITE_ACCESS; + + *numOfVnodes = 0; + struct dirent *de = NULL; + while ((de = readdir(dir)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + if (de->d_type & DT_DIR) { + if (strncmp("vnode", de->d_name, 5) != 0) continue; + int32_t vnode = atoi(de->d_name + 5); + if (vnode == 0) continue; + + (*numOfVnodes)++; + + if (*numOfVnodes >= TSDB_MAX_VNODES) { + dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES); + closedir(dir); + return TSDB_CODE_DND_TOO_MANY_VNODES; + } else { + vnodeList[*numOfVnodes - 1] = vnode; + } + } + } + closedir(dir); + + return TSDB_CODE_SUCCESS; +} + +static void *dnodeOpenVnode(void *param) { + SOpenVnodeThread *pThread = param; + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + + dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + int32_t vgId = pThread->vnodeList[v]; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", vgId, tsOpenVnodes, tsTotalVnodes); + dnodeReportStep("open-vnodes", stepDesc, 0); + + if (vnodeOpen(vgId) < 0) { + dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); + pThread->failed++; + } else { + dDebug("vgId:%d, is opened by thread:%d", vgId, pThread->threadIndex); + pThread->opened++; + } + + atomic_add_fetch_32(&tsOpenVnodes, 1); + } + + dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +int32_t dnodeInitVnodes() { + int32_t vnodeList[TSDB_MAX_VNODES] = {0}; + int32_t numOfVnodes = 0; + int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); + tsTotalVnodes = numOfVnodes; + + if (status != TSDB_CODE_SUCCESS) { + dInfo("get dnode list failed"); + return status; + } + + int32_t threadNum = tsNumOfCores; + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SOpenVnodeThread *pThread = &threads[t]; + pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; + } + + dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) { + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + } + + int32_t openVnodes = 0; + int32_t failedVnodes = 0; + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + pthread_join(pThread->thread, NULL); + } + openVnodes += pThread->opened; + failedVnodes += pThread->failed; + free(pThread->vnodeList); + } + + free(threads); + dInfo("there are total vnodes:%d, opened:%d", numOfVnodes, openVnodes); + + if (failedVnodes != 0) { + dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes); + return -1; + } + + return TSDB_CODE_SUCCESS; +} + +void dnodeCleanupVnodes() { + int32_t vnodeList[TSDB_MAX_VNODES]= {0}; + int32_t numOfVnodes = 0; + int32_t status; + + status = vnodeGetVnodeList(vnodeList, &numOfVnodes); + + if (status != TSDB_CODE_SUCCESS) { + dInfo("get dnode list failed"); + return; + } + + for (int32_t i = 0; i < numOfVnodes; ++i) { + vnodeClose(vnodeList[i]); + } + + dInfo("total vnodes:%d are all closed", numOfVnodes); +} + +static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { + if (pMsg->code != TSDB_CODE_SUCCESS) { + dError("status rsp is received, error:%s", tstrerror(pMsg->code)); + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); + return; + } + + SStatusRsp *pStatusRsp = pMsg->pCont; + SMInfos *minfos = &pStatusRsp->mnodes; + dnodeUpdateMInfos(minfos); + + SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; + pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); + pCfg->moduleStatus = htonl(pCfg->moduleStatus); + pCfg->dnodeId = htonl(pCfg->dnodeId); + dnodeUpdateCfg(pCfg); + + vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes); + + SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); + dnodeUpdateEps(pEps); + + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); +} + +static void dnodeSendStatusMsg(void *handle, void *tmrId) { + if (tsDnodeTmr == NULL) { + dError("dnode timer is already released"); + return; + } + + if (tsStatusTimer == NULL) { + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); + dError("failed to start status timer"); + return; + } + + int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); + SStatusMsg *pStatus = rpcMallocCont(contLen); + if (pStatus == NULL) { + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); + dError("failed to malloc status message"); + return; + } + + dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId); + pStatus->dnodeId = htonl(dnodeGetDnodeId()); + pStatus->version = htonl(tsVersion); + pStatus->lastReboot = htonl(tsRebootTime); + pStatus->numOfCores = htons((uint16_t) tsNumOfCores); + pStatus->diskAvailable = tsAvailDataDirGB; + pStatus->alternativeRole = tsAlternativeRole; + tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); + + // fill cluster cfg parameters + pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes); + pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum); + pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold); + pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); + pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode); + pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); + tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN); + tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); + pStatus->clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); + + pStatus->clusterCfg.enableBalance = tsEnableBalance; + pStatus->clusterCfg.flowCtrl = tsEnableFlowCtrl; + pStatus->clusterCfg.slaveQuery = tsEnableSlaveQuery; + pStatus->clusterCfg.adjustMaster = tsEnableAdjustMaster; + + vnodeBuildStatusMsg(pStatus); + contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); + pStatus->openVnodes = htons(pStatus->openVnodes); + + SRpcMsg rpcMsg = { + .pCont = pStatus, + .contLen = contLen, + .msgType = TSDB_MSG_TYPE_DM_STATUS + }; + + SRpcEpSet epSet; + dnodeGetEpSetForPeer(&epSet); + dnodeSendMsgToDnode(&epSet, &rpcMsg); +} + +void dnodeSendStatusMsgToMnode() { + if (tsDnodeTmr != NULL && tsStatusTimer != NULL) { + dInfo("force send status msg to mnode"); + taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer); + } +} diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 1efaa4a24bc848c99ac265f07ad340f09914ca30..5ecaf19f61a022bae849c2f946acb0ee693aeb59 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -29,13 +29,6 @@ typedef struct { int32_t httpReqNum; } SStatisInfo; -typedef enum { - TSDB_RUN_STATUS_INITIALIZE, - TSDB_RUN_STATUS_RUNING, - TSDB_RUN_STATUS_STOPPED -} SRunStatus; - -SRunStatus dnodeGetRunStatus(); SStatisInfo dnodeGetStatisInfo(); bool dnodeIsFirstDeploy(); @@ -43,9 +36,11 @@ bool dnodeIsMasterEp(char *ep); void dnodeGetEpSetForPeer(SRpcEpSet *epSet); void dnodeGetEpSetForShell(SRpcEpSet *epSet); int32_t dnodeGetDnodeId(); +void dnodeGetClusterId(char *clusterId); + void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); -bool dnodeStartMnode(SMInfos *pMinfos); +int32_t dnodeStartMnode(SMInfos *pMinfos); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); @@ -56,8 +51,10 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid); void *dnodeAllocVWriteQueue(void *pVnode); void dnodeFreeVWriteQueue(void *pWqueue); void dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code); -void *dnodeAllocVReadQueue(void *pVnode); -void dnodeFreeVReadQueue(void *pRqueue); +void *dnodeAllocVQueryQueue(void *pVnode); +void *dnodeAllocVFetchQueue(void *pVnode); +void dnodeFreeVQueryQueue(void *pQqueue); +void dnodeFreeVFetchQueue(void *pFqueue); int32_t dnodeAllocateMPeerQueue(); void dnodeFreeMPeerQueue(); @@ -71,6 +68,16 @@ void dnodeDelayReprocessMWriteMsg(void *pMsg); void dnodeSendStatusMsgToMnode(); +typedef struct { + char *name; + int32_t (*initFp)(); + void (*cleanupFp)(); +} SStep; + +int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize); +void dnodeStepCleanup(SStep *pSteps, int32_t stepSize); +void dnodeReportStep(char *name, char *desc, int8_t finished); + #ifdef __cplusplus } #endif diff --git a/src/inc/mnode.h b/src/inc/mnode.h index bdc30b0c46ced0961715bd48623fdb9e52fb440e..2495a42ba2e5d23cb361e2d64de04d1f710764ea 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -42,11 +42,12 @@ typedef struct SMnodeMsg { struct SVgObj * pVgroup; struct STableObj *pTable; struct SSTableObj*pSTable; + struct SMnodeMsg *pBatchMasterMsg; SMnodeRsp rpcRsp; - int8_t received; - int8_t successed; - int8_t expected; - int8_t retry; + int16_t received; + int16_t successed; + int16_t expected; + int16_t retry; int32_t incomingTs; int32_t code; void * pObj; @@ -57,13 +58,14 @@ typedef struct SMnodeMsg { void * mnodeCreateMsg(SRpcMsg *pRpcMsg); int32_t mnodeInitMsg(SMnodeMsg *pMsg); void mnodeCleanupMsg(SMnodeMsg *pMsg); +void mnodeDestroySubMsg(SMnodeMsg *pSubMsg); int32_t mnodeInitSystem(); int32_t mnodeStartSystem(); void mnodeCleanupSystem(); void mnodeStopSystem(); void sdbUpdateAsync(); -void sdbUpdateSync(void *pMnodes); +int32_t sdbUpdateSync(void *pMnodes); bool mnodeIsRunning(); int32_t mnodeProcessRead(SMnodeMsg *pMsg); int32_t mnodeProcessWrite(SMnodeMsg *pMsg); diff --git a/src/inc/query.h b/src/inc/query.h index 5e1de77889cc469566cc94b729c55622e5462bd6..7342221cb9de1b632ad0f398f2f3a8d27621747a 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -86,6 +86,7 @@ void qDestroyQueryInfo(qinfo_t qHandle); void* qOpenQueryMgmt(int32_t vgId); void qQueryMgmtNotifyClosed(void* pExecutor); +void qQueryMgmtReOpen(void *pExecutor); void qCleanupQueryMgmt(void* pExecutor); void** qRegisterQInfo(void* pMgmt, uint64_t qInfo); void** qAcquireQInfo(void* pMgmt, uint64_t key); diff --git a/src/inc/taos.h b/src/inc/taos.h index cd863587a635ea0aee1e4a288f617e4601b0a09a..cd8e116053bd9adabda9a1eeeb20c6d92679d99d 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -42,6 +42,10 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_BINARY 8 // string #define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes #define TSDB_DATA_TYPE_NCHAR 10 // unicode string +#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte +#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes +#define TSDB_DATA_TYPE_UINT 13 // 4 bytes +#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes typedef enum { TSDB_OPTION_LOCALE, @@ -64,7 +68,7 @@ typedef struct taosField { #define DLL_EXPORT #endif -DLL_EXPORT void taos_init(); +DLL_EXPORT int taos_init(); DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); @@ -136,7 +140,7 @@ DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); -DLL_EXPORT void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param); +//DLL_EXPORT void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ca202933921395c08865b6a29d6a95a5c34b8dec..f4712198ee9c2aee97eda1a03085032c81c53b84 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -36,31 +36,8 @@ extern "C" { #define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) #define TSKEY_INITIAL_VAL INT64_MIN -// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR -typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; - -typedef struct tstr { - VarDataLenT len; - char data[]; -} tstr; - -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) - -#define varDataLen(v) ((VarDataLenT *)(v))[0] -#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) -#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) -#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) -#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) - -// this data type is internally used only in 'in' query to hold the values -#define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) - - // Bytes for each type. -extern const int32_t TYPE_BYTES[11]; +extern const int32_t TYPE_BYTES[15]; // TODO: replace and remove code below #define CHAR_BYTES sizeof(char) @@ -92,6 +69,11 @@ extern const int32_t TYPE_BYTES[11]; #define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF #define TSDB_DATA_BINARY_NULL 0xFF +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + #define TSDB_DATA_NULL_STR "NULL" #define TSDB_DATA_NULL_STR_L "null" @@ -131,19 +113,16 @@ do { \ (src) = (void *)((char *)src + sizeof(type));\ } while(0) -#define GET_INT8_VAL(x) (*(int8_t *)(x)) -#define GET_INT16_VAL(x) (*(int16_t *)(x)) -#define GET_INT32_VAL(x) (*(int32_t *)(x)) -#define GET_INT64_VAL(x) (*(int64_t *)(x)) -#ifdef _TD_ARM_32 - - //#define __float_align_declear() float __underlyFloat = 0.0; - //#define __float_align_declear() - //#define GET_FLOAT_VAL_ALIGN(x) (*(int32_t*)&(__underlyFloat) = *(int32_t*)(x); __underlyFloat); - // notes: src must be float or double type variable !!! - //#define SET_FLOAT_VAL_ALIGN(dst, src) (*(int32_t*) dst = *(int32_t*)src); - //#define SET_DOUBLE_VAL_ALIGN(dst, src) (*(int64_t*) dst = *(int64_t*)src); +#define GET_INT8_VAL(x) (*(int8_t *)(x)) +#define GET_INT16_VAL(x) (*(int16_t *)(x)) +#define GET_INT32_VAL(x) (*(int32_t *)(x)) +#define GET_INT64_VAL(x) (*(int64_t *)(x)) +#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) +#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) +#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) +#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) +#ifdef _TD_ARM_32 float taos_align_get_float(const char* pBuf); double taos_align_get_double(const char* pBuf); @@ -162,57 +141,6 @@ do { \ #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } #endif -typedef struct tDataTypeDescriptor { - int16_t nType; - int16_t nameLen; - int32_t nSize; - char * aName; - int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize); - int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize); - void (*getStatisFunc)(const TSKEY *primaryKey, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); -} tDataTypeDescriptor; - -extern tDataTypeDescriptor tDataTypeDesc[11]; - -bool isValidDataType(int32_t type); -//bool isNull(const char *val, int32_t type); -static FORCE_INLINE bool isNull(const char *val, int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; - case TSDB_DATA_TYPE_TINYINT: - return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; - case TSDB_DATA_TYPE_SMALLINT: - return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; - case TSDB_DATA_TYPE_INT: - return *(uint32_t *)val == TSDB_DATA_INT_NULL; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; - case TSDB_DATA_TYPE_FLOAT: - return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; - case TSDB_DATA_TYPE_DOUBLE: - return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; - case TSDB_DATA_TYPE_NCHAR: - return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; - case TSDB_DATA_TYPE_BINARY: - return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; - default: - return false; - }; -} - -void setVardataNull(char* val, int32_t type); -void setNull(char *val, int32_t type, int32_t bytes); -void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -void* getNullValue(int32_t type); - -void assignVal(char *val, const char *src, int32_t len, int32_t type); -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); - // TODO: check if below is necessary #define TSDB_RELATION_INVALID 0 #define TSDB_RELATION_LESS 1 @@ -235,16 +163,19 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_BINARY_OP_MULTIPLY 32 #define TSDB_BINARY_OP_DIVIDE 33 #define TSDB_BINARY_OP_REMAINDER 34 + + +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) <= TSDB_RELATION_IN)) +#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) + #define TS_PATH_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 #define TSDB_USER_LEN TSDB_UNI_LEN // ACCOUNT is a 32 bit positive integer -// this is the length of its string representation -// including the terminator zero -#define TSDB_ACCT_LEN 11 -#define TSDB_PASSWORD_LEN TSDB_UNI_LEN +// this is the length of its string representation, including the terminator zero +#define TSDB_ACCT_ID_LEN 11 #define TSDB_MAX_COLUMNS 1024 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns @@ -252,12 +183,12 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 33 -#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) +#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_SQL_SHOW_LEN 512 -#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024U) // sql length should be less than 1mb +#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb #define TSDB_APPNAME_LEN TSDB_UNI_LEN @@ -278,14 +209,12 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_EP_LEN (TSDB_FQDN_LEN+6) #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 -#define TSDB_METER_VNODE_BITS 20 -#define TSDB_METER_SID_MASK 0xFFFFF -#define TSDB_SHELL_VNODE_BITS 24 -#define TSDB_SHELL_SID_MASK 0xFF -#define TSDB_HTTP_TOKEN_LEN 20 #define TSDB_SHOW_SQL_LEN 512 #define TSDB_SLOW_QUERY_SQL_LEN 512 +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 + #define TSDB_MQTT_HOSTNAME_LEN 64 #define TSDB_MQTT_PORT_LEN 8 #define TSDB_MQTT_USER_LEN 24 @@ -293,9 +222,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MQTT_TOPIC_LEN 64 #define TSDB_MQTT_CLIENT_ID_LEN 32 -#define TSDB_METER_STATE_OFFLINE 0 -#define TSDB_METER_STATE_ONLLINE 1 - #define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE @@ -314,8 +240,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_REPLICA 5 #define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_BLOCK_DIST_COLUMN_INDEX (-2) #define TSDB_UD_COLUMN_INDEX (-100) -#define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta + +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 #define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode @@ -358,7 +286,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_COMP_LEVEL 2 #define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_WAL_LEVEL 1 +#define TSDB_MIN_WAL_LEVEL 0 #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 @@ -366,6 +294,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_DB_UPDATE 1 #define TSDB_DEFAULT_DB_UPDATE_OPTION 0 +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 1 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 + #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second @@ -373,6 +305,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MIN_DB_REPLICA_OPTION 1 #define TSDB_MAX_DB_REPLICA_OPTION 3 #define TSDB_DEFAULT_DB_REPLICA_OPTION 1 + +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 #define TSDB_DEFAULT_DB_QUORUM_OPTION 1 #define TSDB_MAX_JOIN_TABLE_NUM 5 @@ -384,8 +319,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_RPC_THREADS 5 -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode /* * 1. ordinary sub query for select * from super_table @@ -405,31 +340,31 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u #define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) -#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) -#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) -#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) - -#define TSDB_ORDER_ASC 1 -#define TSDB_ORDER_DESC 2 - -#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 -#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 -#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 -#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 -#define TSDB_DEFAULT_USERS_HASH_SIZE 20 -#define TSDB_DEFAULT_DBS_HASH_SIZE 100 -#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 -#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 -#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 - -#define TSDB_PORT_DNODESHELL 0 -#define TSDB_PORT_DNODEDNODE 5 -#define TSDB_PORT_SYNC 10 -#define TSDB_PORT_HTTP 11 -#define TSDB_PORT_ARBITRATOR 12 - -#define TSDB_MAX_WAL_SIZE (1024*1024) +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) + +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 + +#define TSDB_PORT_DNODESHELL 0 +#define TSDB_PORT_DNODEDNODE 5 +#define TSDB_PORT_SYNC 10 +#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 + +#define TSDB_MAX_WAL_SIZE (1024*1024*3) typedef enum { TAOS_QTYPE_RPC = 0, @@ -439,6 +374,10 @@ typedef enum { TAOS_QTYPE_QUERY = 4 } EQType; +#define TSDB_MAX_TIERS 3 +#define TSDB_MAX_DISKS_PER_TIER 16 +#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) + typedef enum { TSDB_SUPER_TABLE = 0, // super table TSDB_CHILD_TABLE = 1, // table created from super table diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index a720b68e594329bd81082b2885fffc61e1f7f8d8..3eb197868b26115ff25206cc8db2bdd809c634cc 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -23,11 +23,7 @@ extern "C" { #include #include -#ifdef TAOS_ERROR_C -#define TAOS_DEFINE_ERROR(name, mod, code, msg) {.val = (int32_t)((0x80000000 | ((mod)<<16) | (code))), .str=(msg)}, -#else -#define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = (int32_t)((0x80000000 | ((mod)<<16) | (code))); -#endif +#define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) #define TAOS_SUCCEEDED(err) ((err) >= 0) @@ -40,364 +36,388 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SUCCESS 0 -#ifdef TAOS_ERROR_C -static STaosError errors[] = { - {.val = 0, .str = "success"}, -#endif - // rpc -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ACTION_IN_PROGRESS, 0, 0x0001, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_REQUIRED, 0, 0x0002, "Authentication required") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, 0, 0x0003, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, 0, 0x0004, "Redirect") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NOT_READY, 0, 0x0005, "System not ready") // peer is not ready to process data -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ALREADY_PROCESSED, 0, 0x0006, "Message already processed") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED,0, 0x0007, "Last session not finished") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MISMATCHED_LINK_ID, 0, 0x0008, "Mismatched meter id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TOO_SLOW, 0, 0x0009, "Processing of request timed out") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, 0, 0x000A, "Number of sessions reached limit") // too many sessions -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, 0, 0x000B, "Unable to establish connection") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_APP_ERROR, 0, 0x000C, "Unexpected generic error in RPC") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_UNEXPECTED_RESPONSE, 0, 0x000D, "Unexpected response") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VALUE, 0, 0x000E, "Invalid value") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID, 0, 0x000F, "Invalid transaction id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, 0, 0x0010, "Invalid session id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid message type") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Invalid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") +#define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //"Action in progress") +#define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //"Authentication required") +#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //"Authentication failure") +#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //"Redirect") +#define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //"System not ready") // peer is not ready to process data +#define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //"Message already processed") +#define TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x0007) //"Last session not finished") +#define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //"Mismatched meter id") +#define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //"Processing of request timed out") +#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //"Number of sessions reached limit") // too many sessions +#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) //"Unable to establish connection") +#define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //"Unexpected generic error in RPC") +#define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //"Unexpected response") +#define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //"Invalid value") +#define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //"Invalid transaction id") +#define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //"Invalid session id") +#define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //"Invalid message type") +#define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //"Invalid response type") +#define TSDB_CODE_RPC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0013) //"Client and server's time is not synchronized") +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) //"Database not ready") +#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) //"Unable to resolve FQDN") +#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //"Invalid app version") //common & util -TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, 0, 0x0101, "Memory corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, 0, 0x0102, "Out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, 0, 0x0103, "Invalid config message") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, 0, 0x0104, "Data file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, 0, 0x0105, "Ref out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, 0, 0x0106, "too many Ref Objs") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, 0, 0x0107, "Ref ID is removed") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, 0, 0x0108, "Invalid Ref ID") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, 0, 0x0109, "Ref is already there") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, 0, 0x010A, "Ref is not there") +#define TSDB_CODE_COM_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) //"Operation not supported") +#define TSDB_CODE_COM_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) //"Memory corrupted") +#define TSDB_CODE_COM_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102) //"Out of memory") +#define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) //"Invalid config message") +#define TSDB_CODE_COM_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) //"Data file corrupted") +#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) //"Ref out of memory") +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) //"too many Ref Objs") +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) //"Ref ID is removed") +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) //"Invalid Ref ID") +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) //"Ref is already there") +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) //"Ref is not there") //client -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, 0, 0x0200, "Invalid SQL statement") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE, 0, 0x0201, "Invalid qhandle") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TIME_STAMP, 0, 0x0202, "Invalid combination of client/service time") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VALUE, 0, 0x0203, "Invalid value in client") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VERSION, 0, 0x0204, "Invalid client version") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_IE, 0, 0x0205, "Invalid client ie") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_FQDN, 0, 0x0206, "Invalid host name") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_USER_LENGTH, 0, 0x0207, "Invalid user name") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH, 0, 0x0208, "Invalid password") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, 0, 0x0209, "Database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, 0, 0x020A, "Table name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, 0, 0x020B, "Invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, 0, 0x020C, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_DISKSPACE, 0, 0x020D, "System out of disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, 0, 0x020E, "Query cache erased") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, 0, 0x020F, "Query terminated") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, 0, 0x0210, "Result set too large to be sorted") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Application error") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax error in SQL") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, 0, 0x0217, "Database not specified or available") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, 0, 0x0218, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, 0, 0x0219, "SQL statement too long, check maxSQLLength config") +#define TSDB_CODE_TSC_INVALID_SQL TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid SQL statement") +#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) //"Invalid qhandle") +#define TSDB_CODE_TSC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0202) //"Invalid combination of client/service time") +#define TSDB_CODE_TSC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x0203) //"Invalid value in client") +#define TSDB_CODE_TSC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0204) //"Invalid client version") +#define TSDB_CODE_TSC_INVALID_IE TAOS_DEF_ERROR_CODE(0, 0x0205) //"Invalid client ie") +#define TSDB_CODE_TSC_INVALID_FQDN TAOS_DEF_ERROR_CODE(0, 0x0206) //"Invalid host name") +#define TSDB_CODE_TSC_INVALID_USER_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0207) //"Invalid user name") +#define TSDB_CODE_TSC_INVALID_PASS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0208) //"Invalid password") +#define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) //"Database name too long") +#define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) //"Table name too long") +#define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) //"Invalid connection") +#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) //"System out of memory") +#define TSDB_CODE_TSC_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x020D) //"System out of disk space") +#define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) //"Query cache erased") +#define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) //"Query terminated") +#define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) //"Result set too large to be sorted") // too many result for ordered super table projection query +#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) //"Application error") +#define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) //"Action in progress") +#define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) //"Disconnected from service") +#define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) //"No write permission") +#define TSDB_CODE_TSC_CONN_KILLED TAOS_DEF_ERROR_CODE(0, 0x0215) //"Connection killed") +#define TSDB_CODE_TSC_SQL_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x0216) //"Syntax error in SQL") +#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) //"Database not specified or available") +#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) //"Table does not exist") +#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) //"SQL statement too long check maxSQLLength config") // mnode -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, 0, 0x0301, "Message is progressing") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, 0, 0x0302, "Messag need to be reprocessed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, 0, 0x0303, "Insufficient privilege for operation") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, 0, 0x0304, "Unexpected generic error in mnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, 0, 0x0305, "Invalid message connection") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION, 0, 0x0306, "Incompatible protocol version") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN, 0, 0x0307, "Invalid message length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE, 0, 0x0308, "Invalid message type") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS, 0, 0x0309, "Too many connections") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_OUT_OF_MEMORY, 0, 0x030A, "Out of memory in mnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, 0, 0x030B, "Data expired") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, 0, 0x030C, "Invalid query id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, 0, 0x030D, "Invalid stream id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, 0, 0x030E, "Invalid connection id") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "Object already there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "Unexpected generic error in sdb") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE, 0, 0x0322, "Invalid table type") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE, 0, 0x0323, "Object not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW, 0, 0x0324, "Invalid meta row") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE, 0, 0x0325, "Invalid key type") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, 0, 0x0330, "DNode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, 0, 0x0331, "DNode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, 0, 0x0332, "VGroup does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, 0, 0x0333, "Master DNode cannot be removed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, 0, 0x0334, "Out of DNodes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "Cluster cfg inconsistent") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "Invalid dnode cfg option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, 0, 0x0337, "Balance already enabled") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, 0, 0x0338, "Vgroup not in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, 0, 0x0339, "Vgroup already in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, 0, 0x033A, "Dnode not avaliable") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, 0, 0x033B, "Cluster id not match") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, 0, 0x033C, "Cluster not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED, 0, 0x033D, "Dnode Id not configured") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED, 0, 0x033E, "Dnode Ep not configured") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "Account already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "Invalid account") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, 0, 0x0342, "Invalid account options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, 0, 0x0343, "Account authorization has expired") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, 0, 0x0350, "User already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, 0, 0x0351, "Invalid user") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT, 0, 0x0352, "Invalid user format") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, 0, 0x0353, "Invalid password format") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, 0, 0x0354, "Can not get user from conn") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, 0, 0x0355, "Too many users") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST, 0, 0x0360, "Table already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, 0, 0x0361, "Table name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, 0, 0x0362, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, 0, 0x0363, "Invalid table type in tsdb") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, 0, 0x0364, "Too many tags") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, 0, 0x0366, "Too many time series") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, 0, 0x0367, "Not super table") // operation only available for super table -TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, 0, 0x0368, "Tag name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, 0, 0x0369, "Tag already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, 0, 0x036A, "Tag does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, 0, 0x036B, "Field already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, 0, 0x036C, "Field does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, 0, 0x036D, "Super table does not exist") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, 0, 0x0380, "Database not specified or available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, 0, 0x0381, "Database already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, 0, 0x0382, "Invalid database options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "Invalid database name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "Cannot delete monitor database") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many databases for account") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, 0, 0x0390, "Invalid database option: days out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, 0, 0x0391, "Invalid database option: keep >= keep2 >= keep1 >= days") +#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") +#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) //"Message is progressing") +#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) //"Messag need to be reprocessed") +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) //"Insufficient privilege for operation") +#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) //"Unexpected generic error in mnode") +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) //"Invalid message connection") +#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) //"Incompatible protocol version") +#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) //"Invalid message length") +#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) //"Invalid message type") +#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) //"Too many connections") +#define TSDB_CODE_MND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x030A) //"Out of memory in mnode") +#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) //"Data expired") +#define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C) //"Invalid query id") +#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) //"Invalid stream id") +#define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x030E) //"Invalid connection id") +#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0310) //"mnode is alreay running") +#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0311) //"failed to config sync") +#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0312) //"failed to start sync") +#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313) //"failed to create mnode dir") +#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) //"failed to init components") + +#define TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) //"Object already there") +#define TSDB_CODE_MND_SDB_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) //"Unexpected generic error in sdb") +#define TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) //"Invalid table type") +#define TSDB_CODE_MND_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0323) //"Object not there") +#define TSDB_CODE_MND_SDB_INVAID_META_ROW TAOS_DEF_ERROR_CODE(0, 0x0324) //"Invalid meta row") +#define TSDB_CODE_MND_SDB_INVAID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) //"Invalid key type") + +#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) //"DNode already exists") +#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) //"DNode does not exist") +#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0332) //"VGroup does not exist") +#define TSDB_CODE_MND_NO_REMOVE_MASTER TAOS_DEF_ERROR_CODE(0, 0x0333) //"Master DNode cannot be removed") +#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0334) //"Out of DNodes") +#define TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT TAOS_DEF_ERROR_CODE(0, 0x0335) //"Cluster cfg inconsistent") +#define TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION TAOS_DEF_ERROR_CODE(0, 0x0336) //"Invalid dnode cfg option") +#define TSDB_CODE_MND_BALANCE_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0337) //"Balance already enabled") +#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0338) //"Vgroup not in dnode") +#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0339) //"Vgroup already in dnode") +#define TSDB_CODE_MND_DNODE_NOT_FREE TAOS_DEF_ERROR_CODE(0, 0x033A) //"Dnode not avaliable") +#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x033B) //"Cluster id not match") +#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) //"Cluster not ready") +#define TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033D) //"Dnode Id not configured") +#define TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033E) //"Dnode Ep not configured") + +#define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) //"Account already exists") +#define TSDB_CODE_MND_INVALID_ACCT TAOS_DEF_ERROR_CODE(0, 0x0341) //"Invalid account") +#define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0342) //"Invalid account options") +#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0343) //"Account authorization has expired") + +#define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) //"User already exists") +#define TSDB_CODE_MND_INVALID_USER TAOS_DEF_ERROR_CODE(0, 0x0351) //"Invalid user") +#define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0352) //"Invalid user format") +#define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0353) //"Invalid password format") +#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0354) //"Can not get user from conn") +#define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0355) //"Too many users") + +#define TSDB_CODE_MND_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) //"Table already exists") +#define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") +#define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") +#define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) //"Invalid table type in tsdb") +#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) //"Too many tags") +#define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) //"Too many time series") +#define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) //"Not super table") // operation only available for super table +#define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) //"Tag name too long") +#define TSDB_CODE_MND_TAG_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0369) //"Tag already exists") +#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036A) //"Tag does not exist") +#define TSDB_CODE_MND_FIELD_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x036B) //"Field already exists") +#define TSDB_CODE_MND_FIELD_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) //"Field does not exist") +#define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist") +#define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") + +#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") +#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) //"Invalid database name") +#define TSDB_CODE_MND_MONITOR_DB_FORBIDDEN TAOS_DEF_ERROR_CODE(0, 0x0384) //"Cannot delete monitor database") +#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) //"Too many databases for account") +#define TSDB_CODE_MND_DB_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0386) //"Database not available") +#define TSDB_CODE_MND_VGROUP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0387) //"Database unsynced") + +#define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) //"Invalid database option: days out of range") +#define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) //"Invalid database option: keep >= keep1 >= keep0 >= days") // dnode -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length") - -// vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, 0, 0x0501, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, 0, 0x0502, "Action need to be reprocessed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, 0, 0x0503, "Invalid Vgroup ID") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED, 0, 0x0504, "Vnode initialization failed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISKSPACE, 0, 0x0505, "System out of disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS, 0, 0x0506, "No write permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, 0, 0x0507, "Missing data file") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Vnode memory is full because commit failed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Write operation denied") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing") +#define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) //"Message not processed") +#define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) //"Dnode out of memory") +#define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0402) //"No permission for disk files in dnode") +#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0403) //"Invalid message length") +#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0404) //"Action in progress") +#define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") + +// vnode +#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") +#define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") +#define TSDB_CODE_VND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0502) //"Action need to be reprocessed") +#define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) //"Invalid Vgroup ID") +#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) //"Vnode initialization failed") +#define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) //"System out of disk space") +#define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) //"No write permission for disk files") +#define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) //"Missing data file") +#define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) //"Out of memory") +#define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) //"Unexpected generic error in vnode") +#define TSDB_CODE_VND_INVALID_VRESION_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) //"Invalid version file") +#define TSDB_CODE_VND_IS_FULL TAOS_DEF_ERROR_CODE(0, 0x050B) //"Database memory is full for commit failed") +#define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) //"Database memory is full for waiting commit") +#define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) //"Database is dropping") +#define TSDB_CODE_VND_IS_BALANCING TAOS_DEF_ERROR_CODE(0, 0x050E) //"Database is balancing") +#define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) //"Database suspended") +#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied") +#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing") // tsdb -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "Invalid table ID") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, 0, 0x0601, "Invalid table type") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, 0, 0x0602, "Invalid table schema version") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, 0, 0x0603, "Table already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, 0, 0x0604, "Invalid configuration") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, 0, 0x0605, "Tsdb init failed") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, 0, 0x0606, "No diskspace for tsdb") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, 0, 0x0607, "No permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, 0, 0x0608, "Data file(s) corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, 0, 0x0609, "Out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, 0, 0x060A, "Tag too old") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, 0, 0x060B, "Timestamp data out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, 0, 0x060C, "Submit message is messed up") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, 0, 0x060D, "Invalid operation") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "Invalid creation of table") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "No table data in memory skiplist") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, 0, 0x0610, "File already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, 0, 0x0611, "Need to reconfigure table") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, 0, 0x0612, "Invalid information to create table") +#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID") +#define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) //"Invalid table type") +#define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) //"Invalid table schema version") +#define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) //"Table already exists") +#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) //"Invalid configuration") +#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) //"Tsdb init failed") +#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) //"No diskspace for tsdb") +#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) //"No permission for disk files") +#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) //"Data file(s) corrupted") +#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) //"Out of memory") +#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) //"Tag too old") +#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) //"Timestamp data out of range") +#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) //"Submit message is messed up") +#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) //"Invalid operation") +#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) //"Invalid creation of table") +#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) //"No table data in memory skiplist") +#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) //"File already exists") +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) //"Need to reconfigure table") +#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) //"Invalid information to create table") +#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) //"No available disk") +#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") // query -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "Invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, 0, 0x0701, "Invalid message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE, 0, 0x0702, "No diskspace for query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, 0, 0x0703, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "Unexpected generic error in query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "Duplicated join key") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag conditon too many") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached") +#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") +#define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) //"Invalid message") // failed to validate the sql expression msg by vnode +#define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) //"No diskspace for query") +#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) //"System out of memory") +#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) //"Unexpected generic error in query") +#define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) //"Duplicated join key") +#define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) //"Tag conditon too many") +#define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) //"Query not ready") +#define TSDB_CODE_QRY_HAS_RSP TAOS_DEF_ERROR_CODE(0, 0x0708) //"Query should response") +#define TSDB_CODE_QRY_IN_EXEC TAOS_DEF_ERROR_CODE(0, 0x0709) //"Multiple retrieval of this query") +#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query") +#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached") +#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistance in replica") + // grant -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 0x0801, "DNode creation limited by licence") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 0x0802, "Account creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 0x0803, "Table creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 0x0804, "DB creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 0x0805, "User creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 0x0806, "Conn creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 0x0807, "Stream creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 0x0808, "Write speed limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 0x0809, "Storage capacity limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 0x080A, "Query time limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 0x080B, "CPU cores limited by license") +#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") +#define TSDB_CODE_GRANT_DNODE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0801) //"DNode creation limited by licence") +#define TSDB_CODE_GRANT_ACCT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0802) //"Account creation limited by license") +#define TSDB_CODE_GRANT_TIMESERIES_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0803) //"Table creation limited by license") +#define TSDB_CODE_GRANT_DB_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0804) //"DB creation limited by license") +#define TSDB_CODE_GRANT_USER_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0805) //"User creation limited by license") +#define TSDB_CODE_GRANT_CONN_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0806) //"Conn creation limited by license") +#define TSDB_CODE_GRANT_STREAM_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0807) //"Stream creation limited by license") +#define TSDB_CODE_GRANT_SPEED_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0808) //"Write speed limited by license") +#define TSDB_CODE_GRANT_STORAGE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0809) //"Storage capacity limited by license") +#define TSDB_CODE_GRANT_QUERYTIME_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A) //"Query time limited by license") +#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) //"CPU cores limited by license") // sync -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sync Configuration") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") +#define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) //"Invalid Sync Configuration") +#define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) //"Sync module not enabled") +#define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) //"Invalid Sync version") +#define TSDB_CODE_SYN_CONFIRM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0903) //"Sync confirm expired") +#define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) //"Too many sync fwd infos") +#define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) //"Mismatched protocol") +#define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) //"Mismatched clusterId") +#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) //"Mismatched signature") +#define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) //"Invalid msg checksum") +#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") +#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, 0, 0x1002, "WAL size exceeds limit") +#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") +#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") +#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit") // http -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, 0, 0x1101, "url is not support") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVLALID_URL, 0, 0x1102, "invalid url format") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, 0, 0x1103, "no enough memory") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUSET_TOO_BIG, 0, 0x1104, "request size is too big") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_AUTH_INFO, 0, 0x1105, "no auth info input") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_MSG_INPUT, 0, 0x1106, "request is empty") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_SQL_INPUT, 0, 0x1107, "no sql input") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_EXEC_USEDB, 0, 0x1108, "no need to execute use db cmd") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SESSION_FULL, 0, 0x1109, "session list was full") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR, 0, 0x110A, "generate taosd token error") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_MULTI_REQUEST, 0, 0x110B, "size of multi request is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, 0, 0x110C, "failed to create gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, 0, 0x110D, "failed to finish gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_LOGIN_FAILED, 0, 0x110E, "failed to login") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, 0, 0x1120, "invalid http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, 0, 0x1121, "invalid content length") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, 0, 0x1122, "invalid type of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, 0, 0x1123, "invalid format of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, 0, 0x1124, "invalid basic Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, 0, 0x1125, "invalid taosd Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, 0, 0x1126, "failed to parse method") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, 0, 0x1127, "failed to parse target") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, 0, 0x1128, "failed to parse http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, 0, 0x1129, "failed to parse sp") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, 0, 0x112A, "failed to parse status") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, 0, 0x112B, "failed to parse phrase") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, 0, 0x112C, "failed to parse crlf") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, 0, 0x112D, "failed to parse header") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, 0, 0x112E, "failed to parse header key") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, 0, 0x112F, "failed to parse header val") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, 0, 0x1130, "failed to parse chunk size") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, 0, 0x1131, "failed to parse chunk") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, 0, 0x1132, "failed to parse end section") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, 0, 0x1134, "invalid parse state") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, 0, 0x1135, "failed to parse error section") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_NULL, 0, 0x1150, "query size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_SIZE, 0, 0x1151, "query size can not more than 100") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR, 0, 0x1152, "parse grafana json error") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_NOT_INPUT, 0, 0x1160, "database name can not be null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_TOO_LONG, 0, 0x1161, "database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_INVALID_JSON, 0, 0x1162, "invalid telegraf json fromat") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_NULL, 0, 0x1163, "metrics size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_SIZE, 0, 0x1164, "metrics size can not more than 1K") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NULL, 0, 0x1165, "metric name not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_TYPE, 0, 0x1166, "metric name type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_NULL, 0, 0x1167, "metric name length is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_LONG, 0, 0x1168, "metric name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, 0, 0x1169, "timestamp not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, 0, 0x116A, "timestamp type should be integer") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, 0, 0x116B, "timestamp value smaller than 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, 0, 0x116C, "tags not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, 0, 0x116D, "tags size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, 0, 0x116E, "tags size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, 0, 0x116F, "tag is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, 0, 0x1170, "tag name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, 0, 0x1171, "tag name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE, 0, 0x1172, "tag value type should be number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_NULL, 0, 0x1173, "tag value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_NULL, 0, 0x1174, "table is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_SIZE, 0, 0x1175, "table name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_NULL, 0, 0x1176, "fields not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_0, 0, 0x1177, "fields size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG, 0, 0x1178, "fields size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NULL, 0, 0x1179, "field is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_NULL, 0, 0x117A, "field name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE, 0, 0x117B, "field name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE, 0, 0x117C, "field value type should be number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL, 0, 0x117D, "field value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_HOST_NOT_STRING, 0, 0x117E, "host type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST, 0, 0x117F, "stable not exist") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_NOT_INPUT, 0, 0x1190, "database name can not be null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_TOO_LONG, 0, 0x1191, "database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_INVALID_JSON, 0, 0x1192, "invalid opentsdb json fromat") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_NULL, 0, 0x1193, "metrics size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_SIZE, 0, 0x1194, "metrics size can not more than 10K") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NULL, 0, 0x1195, "metric name not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_TYPE, 0, 0x1196, "metric name type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_NULL, 0, 0x1197, "metric name length is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_LONG, 0, 0x1198, "metric name length can not more than 22") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_NULL, 0, 0x1199, "timestamp not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE, 0, 0x119A, "timestamp type should be integer") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL, 0, 0x119B, "timestamp value smaller than 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_NULL, 0, 0x119C, "tags not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_0, 0, 0x119D, "tags size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG, 0, 0x119E, "tags size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NULL, 0, 0x119F, "tag is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_NULL, 0, 0x11A0, "tag name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_SIZE, 0, 0x11A1, "tag name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, 0, 0x11A2, "tag value type should be boolean, number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, 0, 0x11A3, "tag value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value can not more than 64") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string") +#define TSDB_CODE_HTTP_SERVER_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x1100) //"http server is not onlin") +#define TSDB_CODE_HTTP_UNSUPPORT_URL TAOS_DEF_ERROR_CODE(0, 0x1101) //"url is not support") +#define TSDB_CODE_HTTP_INVALID_URL TAOS_DEF_ERROR_CODE(0, 0x1102) //invalid url format") +#define TSDB_CODE_HTTP_NO_ENOUGH_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1103) //"no enough memory") +#define TSDB_CODE_HTTP_REQUSET_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x1104) //"request size is too big") +#define TSDB_CODE_HTTP_NO_AUTH_INFO TAOS_DEF_ERROR_CODE(0, 0x1105) //"no auth info input") +#define TSDB_CODE_HTTP_NO_MSG_INPUT TAOS_DEF_ERROR_CODE(0, 0x1106) //"request is empty") +#define TSDB_CODE_HTTP_NO_SQL_INPUT TAOS_DEF_ERROR_CODE(0, 0x1107) //"no sql input") +#define TSDB_CODE_HTTP_NO_EXEC_USEDB TAOS_DEF_ERROR_CODE(0, 0x1108) //"no need to execute use db cmd") +#define TSDB_CODE_HTTP_SESSION_FULL TAOS_DEF_ERROR_CODE(0, 0x1109) //"session list was full") +#define TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR TAOS_DEF_ERROR_CODE(0, 0x110A) //"generate taosd token error") +#define TSDB_CODE_HTTP_INVALID_MULTI_REQUEST TAOS_DEF_ERROR_CODE(0, 0x110B) //"size of multi request is 0") +#define TSDB_CODE_HTTP_CREATE_GZIP_FAILED TAOS_DEF_ERROR_CODE(0, 0x110C) //"failed to create gzip") +#define TSDB_CODE_HTTP_FINISH_GZIP_FAILED TAOS_DEF_ERROR_CODE(0, 0x110D) //"failed to finish gzip") +#define TSDB_CODE_HTTP_LOGIN_FAILED TAOS_DEF_ERROR_CODE(0, 0x110E) //"failed to login") + +#define TSDB_CODE_HTTP_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x1120) //"invalid http version") +#define TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH TAOS_DEF_ERROR_CODE(0, 0x1121) //"invalid content length") +#define TSDB_CODE_HTTP_INVALID_AUTH_TYPE TAOS_DEF_ERROR_CODE(0, 0x1122) //"invalid type of Authorization") +#define TSDB_CODE_HTTP_INVALID_AUTH_FORMAT TAOS_DEF_ERROR_CODE(0, 0x1123) //"invalid format of Authorization") +#define TSDB_CODE_HTTP_INVALID_BASIC_AUTH TAOS_DEF_ERROR_CODE(0, 0x1124) //"invalid basic Authorization") +#define TSDB_CODE_HTTP_INVALID_TAOSD_AUTH TAOS_DEF_ERROR_CODE(0, 0x1125) //"invalid taosd Authorization") +#define TSDB_CODE_HTTP_PARSE_METHOD_FAILED TAOS_DEF_ERROR_CODE(0, 0x1126) //"failed to parse method") +#define TSDB_CODE_HTTP_PARSE_TARGET_FAILED TAOS_DEF_ERROR_CODE(0, 0x1127) //"failed to parse target") +#define TSDB_CODE_HTTP_PARSE_VERSION_FAILED TAOS_DEF_ERROR_CODE(0, 0x1128) //"failed to parse http version") +#define TSDB_CODE_HTTP_PARSE_SP_FAILED TAOS_DEF_ERROR_CODE(0, 0x1129) //"failed to parse sp") +#define TSDB_CODE_HTTP_PARSE_STATUS_FAILED TAOS_DEF_ERROR_CODE(0, 0x112A) //"failed to parse status") +#define TSDB_CODE_HTTP_PARSE_PHRASE_FAILED TAOS_DEF_ERROR_CODE(0, 0x112B) //"failed to parse phrase") +#define TSDB_CODE_HTTP_PARSE_CRLF_FAILED TAOS_DEF_ERROR_CODE(0, 0x112C) //"failed to parse crlf") +#define TSDB_CODE_HTTP_PARSE_HEADER_FAILED TAOS_DEF_ERROR_CODE(0, 0x112D) //"failed to parse header") +#define TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED TAOS_DEF_ERROR_CODE(0, 0x112E) //"failed to parse header key") +#define TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED TAOS_DEF_ERROR_CODE(0, 0x112F) //"failed to parse header val") +#define TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED TAOS_DEF_ERROR_CODE(0, 0x1130) //"failed to parse chunk size") +#define TSDB_CODE_HTTP_PARSE_CHUNK_FAILED TAOS_DEF_ERROR_CODE(0, 0x1131) //"failed to parse chunk") +#define TSDB_CODE_HTTP_PARSE_END_FAILED TAOS_DEF_ERROR_CODE(0, 0x1132) //"failed to parse end section") +#define TSDB_CODE_HTTP_PARSE_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x1134) //"invalid parse state") +#define TSDB_CODE_HTTP_PARSE_ERROR_STATE TAOS_DEF_ERROR_CODE(0, 0x1135) //"failed to parse error section") + +#define TSDB_CODE_HTTP_GC_QUERY_NULL TAOS_DEF_ERROR_CODE(0, 0x1150) //"query size is 0") +#define TSDB_CODE_HTTP_GC_QUERY_SIZE TAOS_DEF_ERROR_CODE(0, 0x1151) //"query size can not more than 100") +#define TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR TAOS_DEF_ERROR_CODE(0, 0x1152) //"parse grafana json error") + +#define TSDB_CODE_HTTP_TG_DB_NOT_INPUT TAOS_DEF_ERROR_CODE(0, 0x1160) //"database name can not be null") +#define TSDB_CODE_HTTP_TG_DB_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x1161) //"database name too long") +#define TSDB_CODE_HTTP_TG_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x1162) //"invalid telegraf json fromat") +#define TSDB_CODE_HTTP_TG_METRICS_NULL TAOS_DEF_ERROR_CODE(0, 0x1163) //"metrics size is 0") +#define TSDB_CODE_HTTP_TG_METRICS_SIZE TAOS_DEF_ERROR_CODE(0, 0x1164) //"metrics size can not more than 1K") +#define TSDB_CODE_HTTP_TG_METRIC_NULL TAOS_DEF_ERROR_CODE(0, 0x1165) //"metric name not find") +#define TSDB_CODE_HTTP_TG_METRIC_TYPE TAOS_DEF_ERROR_CODE(0, 0x1166) //"metric name type should be string") +#define TSDB_CODE_HTTP_TG_METRIC_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1167) //"metric name length is 0") +#define TSDB_CODE_HTTP_TG_METRIC_NAME_LONG TAOS_DEF_ERROR_CODE(0, 0x1168) //"metric name length too long") +#define TSDB_CODE_HTTP_TG_TIMESTAMP_NULL TAOS_DEF_ERROR_CODE(0, 0x1169) //"timestamp not find") +#define TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE TAOS_DEF_ERROR_CODE(0, 0x116A) //"timestamp type should be integer") +#define TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL TAOS_DEF_ERROR_CODE(0, 0x116B) //"timestamp value smaller than 0") +#define TSDB_CODE_HTTP_TG_TAGS_NULL TAOS_DEF_ERROR_CODE(0, 0x116C) //"tags not find") +#define TSDB_CODE_HTTP_TG_TAGS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x116D) //"tags size is 0") +#define TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x116E) //"tags size too long") +#define TSDB_CODE_HTTP_TG_TAG_NULL TAOS_DEF_ERROR_CODE(0, 0x116F) //"tag is null") +#define TSDB_CODE_HTTP_TG_TAG_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1170) //"tag name is null") +#define TSDB_CODE_HTTP_TG_TAG_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x1171) //"tag name length too long") +#define TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x1172) //"tag value type should be number or string") +#define TSDB_CODE_HTTP_TG_TAG_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x1173) //"tag value is null") +#define TSDB_CODE_HTTP_TG_TABLE_NULL TAOS_DEF_ERROR_CODE(0, 0x1174) //"table is null") +#define TSDB_CODE_HTTP_TG_TABLE_SIZE TAOS_DEF_ERROR_CODE(0, 0x1175) //"table name length too long") +#define TSDB_CODE_HTTP_TG_FIELDS_NULL TAOS_DEF_ERROR_CODE(0, 0x1176) //"fields not find") +#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x1177) //"fields size is 0") +#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x1178) //"fields size too long") +#define TSDB_CODE_HTTP_TG_FIELD_NULL TAOS_DEF_ERROR_CODE(0, 0x1179) //"field is null") +#define TSDB_CODE_HTTP_TG_FIELD_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x117A) //"field name is null") +#define TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x117B) //"field name length too long") +#define TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x117C) //"field value type should be number or string") +#define TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x117D) //"field value is null") +#define TSDB_CODE_HTTP_TG_HOST_NOT_STRING TAOS_DEF_ERROR_CODE(0, 0x117E) //"host type should be string") +#define TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x117F) //"stable not exist") + +#define TSDB_CODE_HTTP_OP_DB_NOT_INPUT TAOS_DEF_ERROR_CODE(0, 0x1190) //"database name can not be null") +#define TSDB_CODE_HTTP_OP_DB_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x1191) //"database name too long") +#define TSDB_CODE_HTTP_OP_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x1192) //"invalid opentsdb json fromat") +#define TSDB_CODE_HTTP_OP_METRICS_NULL TAOS_DEF_ERROR_CODE(0, 0x1193) //"metrics size is 0") +#define TSDB_CODE_HTTP_OP_METRICS_SIZE TAOS_DEF_ERROR_CODE(0, 0x1194) //"metrics size can not more than 10K") +#define TSDB_CODE_HTTP_OP_METRIC_NULL TAOS_DEF_ERROR_CODE(0, 0x1195) //"metric name not find") +#define TSDB_CODE_HTTP_OP_METRIC_TYPE TAOS_DEF_ERROR_CODE(0, 0x1196) //"metric name type should be string") +#define TSDB_CODE_HTTP_OP_METRIC_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1197) //"metric name length is 0") +#define TSDB_CODE_HTTP_OP_METRIC_NAME_LONG TAOS_DEF_ERROR_CODE(0, 0x1198) //"metric name length can not more than 22") +#define TSDB_CODE_HTTP_OP_TIMESTAMP_NULL TAOS_DEF_ERROR_CODE(0, 0x1199) //"timestamp not find") +#define TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE TAOS_DEF_ERROR_CODE(0, 0x119A) //"timestamp type should be integer") +#define TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL TAOS_DEF_ERROR_CODE(0, 0x119B) //"timestamp value smaller than 0") +#define TSDB_CODE_HTTP_OP_TAGS_NULL TAOS_DEF_ERROR_CODE(0, 0x119C) //"tags not find") +#define TSDB_CODE_HTTP_OP_TAGS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x119D) //"tags size is 0") +#define TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x119E) //"tags size too long") +#define TSDB_CODE_HTTP_OP_TAG_NULL TAOS_DEF_ERROR_CODE(0, 0x119F) //"tag is null") +#define TSDB_CODE_HTTP_OP_TAG_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x11A0) //"tag name is null") +#define TSDB_CODE_HTTP_OP_TAG_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x11A1) //"tag name length too long") +#define TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A2) //"tag value type should be boolean number or string") +#define TSDB_CODE_HTTP_OP_TAG_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A3) //"tag value is null") +#define TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x11A4) //"tag value can not more than 64") +#define TSDB_CODE_HTTP_OP_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A5) //"value not find") +#define TSDB_CODE_HTTP_OP_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A6) //"value type should be boolean number or string") // odbc -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, 0, 0x2100, "out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, 0, 0x2101, "convertion not a valid literal input") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, 0, 0x2102, "convertion undefined") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC_FRAC, 0, 0x2103, "convertion fractional truncated") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC, 0, 0x2104, "convertion truncated") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_SUPPORT, 0, 0x2105, "convertion not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_OOR, 0, 0x2106, "convertion numeric value out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OUT_OF_RANGE, 0, 0x2107, "out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NOT_SUPPORT, 0, 0x2108, "not supported yet") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_HANDLE, 0, 0x2109, "invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_RESULT, 0, 0x210a, "no result set") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_FIELDS, 0, 0x210b, "no fields returned") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_CURSOR, 0, 0x210c, "invalid cursor") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_STATEMENT_NOT_READY, 0, 0x210d, "statement not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONNECTION_BUSY, 0, 0x210e, "connection still busy") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_CONNSTR, 0, 0x210f, "bad connection string") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_ARG, 0, 0x2110, "bad argument") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_VALID_TS, 0, 0x2111, "not a valid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE, 0, 0x2112, "src too large") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, 0, 0x2113, "src bad sequence") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, 0, 0x2114, "src incomplete") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, 0, 0x2115, "src general") - -#ifdef TAOS_ERROR_C -}; -#endif - +#define TSDB_CODE_ODBC_OOM TAOS_DEF_ERROR_CODE(0, 0x2100) //"out of memory") +#define TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM TAOS_DEF_ERROR_CODE(0, 0x2101) //"convertion not a valid literal input") +#define TSDB_CODE_ODBC_CONV_UNDEF TAOS_DEF_ERROR_CODE(0, 0x2102) //"convertion undefined") +#define TSDB_CODE_ODBC_CONV_TRUNC_FRAC TAOS_DEF_ERROR_CODE(0, 0x2103) //"convertion fractional truncated") +#define TSDB_CODE_ODBC_CONV_TRUNC TAOS_DEF_ERROR_CODE(0, 0x2104) //"convertion truncated") +#define TSDB_CODE_ODBC_CONV_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x2105) //"convertion not supported") +#define TSDB_CODE_ODBC_CONV_OOR TAOS_DEF_ERROR_CODE(0, 0x2106) //"convertion numeric value out of range") +#define TSDB_CODE_ODBC_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x2107) //"out of range") +#define TSDB_CODE_ODBC_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x2108) //"not supported yet") +#define TSDB_CODE_ODBC_INVALID_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2109) //"invalid handle") +#define TSDB_CODE_ODBC_NO_RESULT TAOS_DEF_ERROR_CODE(0, 0x210a) //"no result set") +#define TSDB_CODE_ODBC_NO_FIELDS TAOS_DEF_ERROR_CODE(0, 0x210b) //"no fields returned") +#define TSDB_CODE_ODBC_INVALID_CURSOR TAOS_DEF_ERROR_CODE(0, 0x210c) //"invalid cursor") +#define TSDB_CODE_ODBC_STATEMENT_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x210d) //"statement not ready") +#define TSDB_CODE_ODBC_CONNECTION_BUSY TAOS_DEF_ERROR_CODE(0, 0x210e) //"connection still busy") +#define TSDB_CODE_ODBC_BAD_CONNSTR TAOS_DEF_ERROR_CODE(0, 0x210f) //"bad connection string") +#define TSDB_CODE_ODBC_BAD_ARG TAOS_DEF_ERROR_CODE(0, 0x2110) //"bad argument") +#define TSDB_CODE_ODBC_CONV_NOT_VALID_TS TAOS_DEF_ERROR_CODE(0, 0x2111) //"not a valid timestamp") +#define TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x2112) //"src too large") +#define TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ TAOS_DEF_ERROR_CODE(0, 0x2113) //"src bad sequence") +#define TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x2114) //"src incomplete") +#define TSDB_CODE_ODBC_CONV_SRC_GENERAL TAOS_DEF_ERROR_CODE(0, 0x2115) //"src general") + +// tfs +#define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") +#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) //"tfs invalid mount config") +#define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) //"tfs too many mount") +#define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) //"tfs duplicate primary mount") +#define TSDB_CODE_FS_NO_PRIMARY_DISK TAOS_DEF_ERROR_CODE(0, 0x2204) //"tfs no primary mount") +#define TSDB_CODE_FS_NO_MOUNT_AT_TIER TAOS_DEF_ERROR_CODE(0, 0x2205) //"tfs no mount at tier") +#define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) //"tfs file already exists") +#define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") +#define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e8e302924490eb0ebcccf10468c3469107fe6dfe..721b9ca6054d7a1fca12304eef33a2a0ce00ddfc 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -105,10 +105,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_AUTH, "auth" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) - - -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" ) - +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) #ifndef TAOS_MESSAGE_C TSDB_MSG_TYPE_MAX // 105 @@ -156,30 +153,31 @@ enum _mgmt_table { #define TSDB_ALTER_TABLE_DROP_COLUMN 6 #define TSDB_ALTER_TABLE_CHANGE_COLUMN 7 -#define TSDB_FILL_NONE 0 -#define TSDB_FILL_NULL 1 -#define TSDB_FILL_SET_VALUE 2 -#define TSDB_FILL_LINEAR 3 -#define TSDB_FILL_PREV 4 +#define TSDB_FILL_NONE 0 +#define TSDB_FILL_NULL 1 +#define TSDB_FILL_SET_VALUE 2 +#define TSDB_FILL_LINEAR 3 +#define TSDB_FILL_PREV 4 +#define TSDB_FILL_NEXT 5 -#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_PASSWD 0x1 #define TSDB_ALTER_USER_PRIVILEGES 0x2 -#define TSDB_KILL_MSG_LEN 30 +#define TSDB_KILL_MSG_LEN 30 -#define TSDB_VN_READ_ACCCESS ((char)0x1) -#define TSDB_VN_WRITE_ACCCESS ((char)0x2) +#define TSDB_VN_READ_ACCCESS ((char)0x1) +#define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u // the normal column of the table -#define TSDB_COL_TAG 0x1u // the tag column type -#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column -#define TSDB_COL_NULL 0x4u // the column filter NULL or not +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_NULL 0x4u // the column filter NULL or not -#define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) -#define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) -#define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) -#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) +#define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) +#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) extern char *taosMsg[]; @@ -263,32 +261,36 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_FNAME_LEN]; - char superTableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; + char stableFname[TSDB_TABLE_FNAME_LEN]; char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + int32_t len; // one create table message + char tableName[TSDB_TABLE_FNAME_LEN]; int8_t igExists; int8_t getMeta; int16_t numOfTags; int16_t numOfColumns; int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string - int32_t contLen; int8_t reserved[16]; char schema[]; +} SCreateTableMsg; + +typedef struct { + int32_t numOfTables; + int32_t contLen; } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_FNAME_LEN]; + char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; int16_t type; /* operation type */ int16_t numOfCols; /* number of schema */ int32_t tagValLen; @@ -320,8 +322,9 @@ typedef struct { } SConnectMsg; typedef struct { - char acctId[TSDB_ACCT_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; char serverVersion[TSDB_VERSION_LEN]; + char clusterId[TSDB_CLUSTER_ID_LEN]; int8_t writeAuth; int8_t superAuth; int8_t reserved1; @@ -366,14 +369,14 @@ typedef struct { int32_t vgId; int32_t tid; uint64_t uid; - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SMDDropTableMsg; typedef struct { int32_t contLen; int32_t vgId; uint64_t uid; - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SDropSTableMsg; typedef struct { @@ -473,10 +476,11 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; uint16_t tagCondLen; // tag length in current query + uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx - int64_t tableLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. + int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. int16_t prjOrder; // global order in super table projection query. int64_t limit; int64_t offset; @@ -491,6 +495,8 @@ typedef struct { int32_t tsNumOfBlocks; // ts comp block numbers int32_t tsOrder; // ts comp block order int32_t numOfTags; // number of tags columns involved + int32_t sqlstrLen; // sql query string + int32_t prevResultLen; // previous result length SColumnInfo colList[]; } SQueryTableMsg; @@ -515,19 +521,21 @@ typedef struct SRetrieveTableRsp { } SRetrieveTableRsp; typedef struct { - int32_t vgId; - int32_t cfgVersion; - int64_t totalStorage; - int64_t compStorage; - int64_t pointsWritten; - uint8_t status; - uint8_t role; - uint8_t replica; - uint8_t reserved[5]; + int32_t vgId; + int32_t dbCfgVersion; + int64_t totalStorage; + int64_t compStorage; + int64_t pointsWritten; + uint64_t vnodeVersion; + int32_t vgCfgVersion; + uint8_t status; + uint8_t role; + uint8_t replica; + uint8_t reserved; } SVnodeLoad; typedef struct { - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; int32_t cacheBlockSize; //MB int32_t totalBlocks; int32_t maxTables; @@ -546,7 +554,8 @@ typedef struct { int8_t quorum; int8_t ignoreExist; int8_t update; - int8_t reserve[9]; + int8_t cacheLastRow; + int8_t reserve[8]; } SCreateDbMsg, SAlterDbMsg; typedef struct { @@ -601,7 +610,6 @@ typedef struct { typedef struct { int32_t numOfMnodes; // tsNumOfMnodes - int32_t enableBalance; // tsEnableBalance int32_t mnodeEqualVnodeNum; // tsMnodeEqualVnodeNum int32_t offlineThreshold; // tsOfflineThreshold int32_t statusInterval; // tsStatusInterval @@ -612,6 +620,11 @@ typedef struct { int64_t checkTime; // 1970-01-01 00:00:00.000 char locale[TSDB_LOCALE_LEN]; // tsLocale char charset[TSDB_LOCALE_LEN]; // tsCharset + int8_t enableBalance; // tsEnableBalance + int8_t flowCtrl; + int8_t slaveQuery; + int8_t adjustMaster; + int8_t reserved[4]; } SClusterCfg; typedef struct { @@ -639,7 +652,7 @@ typedef struct { typedef struct { uint32_t vgId; - int32_t cfgVersion; + int32_t dbCfgVersion; int32_t maxTables; int32_t cacheBlockSize; int32_t totalBlocks; @@ -654,11 +667,14 @@ typedef struct { int8_t precision; int8_t compression; int8_t walLevel; - int8_t replications; + int8_t vgReplica; int8_t wals; int8_t quorum; int8_t update; - int8_t reserved[15]; + int8_t cacheLastRow; + int32_t vgCfgVersion; + int8_t dbReplica; + int8_t reserved[9]; } SVnodeCfg; typedef struct { @@ -667,13 +683,13 @@ typedef struct { } SVnodeDesc; typedef struct { - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; SVnodeCfg cfg; SVnodeDesc nodes[TSDB_MAX_REPLICA]; } SCreateVnodeMsg, SAlterVnodeMsg; typedef struct { - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; int16_t createFlag; char tags[]; } STableInfoMsg; @@ -711,8 +727,7 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_FNAME_LEN]; // table id - char sTableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; // table id uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -722,6 +737,9 @@ typedef struct STableMetaMsg { int32_t tid; uint64_t uid; SVgroupMsg vgroup; + + char sTableName[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; SSchema schema[]; } STableMetaMsg; @@ -733,8 +751,8 @@ typedef struct SMultiTableMeta { typedef struct { int32_t dataLen; - char name[TSDB_TABLE_FNAME_LEN]; - char data[TSDB_MAX_TAGS_LEN + TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * TSDB_MAX_TAGS]; + char name[TSDB_TABLE_FNAME_LEN]; + char *data; } STagData; /* @@ -744,7 +762,7 @@ typedef struct { */ typedef struct { int8_t type; - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; uint16_t payloadLen; char payload[]; } SShowMsg; @@ -790,6 +808,7 @@ typedef struct { typedef struct { char sql[TSDB_SHOW_SQL_LEN]; + char dstTable[TSDB_TABLE_NAME_LEN]; uint32_t streamId; int64_t num; // number of computing/cycles int64_t useconds; @@ -829,7 +848,7 @@ typedef struct { uint64_t uid; uint64_t stime; // stream starting time int32_t status; - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SAlterStreamMsg; typedef struct { @@ -840,6 +859,14 @@ typedef struct { char ckey[TSDB_KEY_LEN]; } SAuthMsg, SAuthRsp; +typedef struct { + int8_t finished; + int8_t reserved1[7]; + char name[TSDB_STEP_NAME_LEN]; + char desc[TSDB_STEP_DESC_LEN]; + char reserved2[64]; +} SStartupStep; + #pragma pack(pop) #ifdef __cplusplus diff --git a/src/inc/tcq.h b/src/inc/tcq.h index afa744a9c4856521075fb9427d0ed1bb412aa7d5..1941649d0a02e7c75f4781cad5dedfa185127752 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -26,7 +26,7 @@ typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg typedef struct { int32_t vgId; char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; + char pass[TSDB_KEY_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; } SCqCfg; @@ -42,7 +42,7 @@ void cqStart(void *handle); void cqStop(void *handle); // cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema); +void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema); // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate void cqDrop(void *handle); diff --git a/src/inc/tfs.h b/src/inc/tfs.h new file mode 100644 index 0000000000000000000000000000000000000000..76e9b17a621f9842479396ada65bb10ac147366c --- /dev/null +++ b/src/inc/tfs.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TD_TFS_H +#define TD_TFS_H + +#include "tglobal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + int level; + int id; +} SDiskID; + +#define TFS_UNDECIDED_LEVEL -1 +#define TFS_UNDECIDED_ID -1 +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 + +// FS APIs ==================================== +typedef struct { + int64_t tsize; + int64_t avail; +} SFSMeta; + +int tfsInit(SDiskCfg *pDiskCfg, int ndisk); +void tfsDestroy(); +void tfsUpdateInfo(SFSMeta *pFSMeta); +void tfsGetMeta(SFSMeta *pMeta); +void tfsAllocDisk(int expLevel, int *level, int *id); + +const char *TFS_PRIMARY_PATH(); +const char *TFS_DISK_PATH(int level, int id); + +// TFILE APIs ==================================== +typedef struct { + int level; + int id; + char rname[TSDB_FILENAME_LEN]; // REL name + char aname[TSDB_FILENAME_LEN]; // ABS name +} TFILE; + +#define TFILE_LEVEL(pf) ((pf)->level) +#define TFILE_ID(pf) ((pf)->id) +#define TFILE_NAME(pf) ((pf)->aname) +#define TFILE_REL_NAME(pf) ((pf)->rname) + +#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags) +#define tfsclose(fd) close(fd) +#define tfsremove(pf) remove(TFILE_NAME(pf)) +#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df)) +#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df)) + +void tfsInitFile(TFILE *pf, int level, int id, const char *bname); +bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); +int tfsEncodeFile(void **buf, TFILE *pf); +void *tfsDecodeFile(void *buf, TFILE *pf); +void tfsbasename(const TFILE *pf, char *dest); +void tfsdirname(const TFILE *pf, char *dest); + +// DIR APIs ==================================== +int tfsMkdirAt(const char *rname, int level, int id); +int tfsMkdirRecurAt(const char *rname, int level, int id); +int tfsMkdir(const char *rname); +int tfsRmdir(const char *rname); +int tfsRename(char *orname, char *nrname); + +typedef struct TDIR TDIR; + +TDIR * tfsOpendir(const char *rname); +const TFILE *tfsReaddir(TDIR *tdir); +void tfsClosedir(TDIR *tdir); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 58859f42bc80daa3317d789950c1625c1533cf5f..78cd2927c7b85d39dd0a965971334e4c3b1cbddd 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -40,7 +40,8 @@ extern "C" { // TSDB STATE DEFINITION #define TSDB_STATE_OK 0x0 -#define TSDB_STATE_BAD_FILE 0x1 +#define TSDB_STATE_BAD_META 0x1 +#define TSDB_STATE_BAD_DATA 0x2 // --------- TSDB APPLICATION HANDLE DEFINITION typedef struct { @@ -48,7 +49,7 @@ typedef struct { void *cqH; int (*notifyStatus)(void *, int status, int eno); int (*eventCallBack)(void *); - void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); + void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema); void (*cqDropFunc)(void *handle); } STsdbAppH; @@ -66,6 +67,7 @@ typedef struct { int8_t precision; int8_t compression; int8_t update; + int8_t cacheLastRow; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS @@ -75,17 +77,17 @@ typedef struct { int64_t pointsWritten; // total data points written } STsdbStat; -typedef void TSDB_REPO_T; // use void to hide implementation details from outside +typedef struct STsdbRepo STsdbRepo; -STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo); +STsdbCfg *tsdbGetCfg(const STsdbRepo *repo); // --------- TSDB REPOSITORY DEFINITION -int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg); -int32_t tsdbDropRepo(char *rootDir); -TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); -int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); -int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg); -int tsdbGetState(TSDB_REPO_T *repo); +int32_t tsdbCreateRepo(int repoid); +int32_t tsdbDropRepo(int repoid); +STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); +int tsdbCloseRepo(STsdbRepo *repo, int toCommit); +int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg); +int tsdbGetState(STsdbRepo *repo); // --------- TSDB TABLE DEFINITION typedef struct { @@ -109,19 +111,20 @@ typedef struct { void tsdbClearTableCfg(STableCfg *config); -void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes); -char* tsdbGetTableName(void *pTable); +void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes); +char *tsdbGetTableName(void *pTable); #define TSDB_TABLEID(_table) ((STableId*) (_table)) +#define TSDB_PREV_ROW 0x1 +#define TSDB_NEXT_ROW 0x2 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); -int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg); -int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); -int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); -TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); +int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg); +int tsdbDropTable(STsdbRepo *pRepo, STableId tableId); +int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg); -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); +uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); // the TSDB repository info typedef struct STsdbRepoInfo { @@ -131,7 +134,7 @@ typedef struct STsdbRepoInfo { int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository // TODO: Other informations to add } STsdbRepoInfo; -STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo); +STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo); // the meter information report structure typedef struct { @@ -140,7 +143,6 @@ typedef struct { int64_t tableTotalDataSize; // In bytes int64_t tableTotalDiskSize; // In bytes } STableInfo; -STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid); // -- FOR INSERT DATA /** @@ -150,7 +152,7 @@ STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid); * * @return the number of points inserted, -1 for failure and the error number is set */ -int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); +int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); // -- FOR QUERY TIME SERIES DATA @@ -159,15 +161,16 @@ typedef void *TsdbQueryHandleT; // Use void to hide implementation details // query condition to build vnode iterator typedef struct STsdbQueryCond { STimeWindow twindow; - int32_t order; // desc|asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo *colList; + bool loadExternalRows; // load external rows or not } STsdbQueryCond; typedef struct SMemRef { - int32_t ref; - void *mem; - void *imem; + int32_t ref; + void * mem; + void * imem; } SMemRef; typedef struct SDataBlockInfo { @@ -179,14 +182,14 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct { - void *pTable; - TSKEY lastKey; + void *pTable; + TSKEY lastKey; } STableKeyInfo; typedef struct { size_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid + SArray * pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; /** @@ -199,7 +202,8 @@ typedef struct { * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef); +TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, + SMemRef *pRef); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -211,14 +215,15 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab * @param tableInfo table list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef); +TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, + SMemRef *pRef); /** * get the queried table object list * @param pHandle * @return */ -SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); +SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); /** * get the group list according to table id from client @@ -228,16 +233,35 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); * @param qinfo * @return */ -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, - void *qinfo, SMemRef* pRef); +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, + void *qinfo, SMemRef *pRef); + + +/** + * get num of rows in mem table + * + * @param pHandle + * @return row size + */ + +int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); /** - * move to next block if exists + * move to next block if exists * * @param pQueryHandle * @return */ bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle); +/** + * move to next block if exists but not merge data in memtable + * + * @param pQueryHandle + * @return + */ +bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT *pQueryHandle); + +SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t type); /** * Get current data block information @@ -246,7 +270,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle); * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo); +void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo); /** * @@ -277,7 +301,7 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL * @param stableid. super table sid * @param pTagCond. tag query condition */ -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); @@ -295,7 +319,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); /** * @@ -304,7 +328,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, ST * @param pGroupInfo * @return */ -int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo); +int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); /** * clean up the query handle @@ -321,12 +345,16 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle); */ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage); -int tsdbInitCommitQueue(int nthreads); +int tsdbInitCommitQueue(); void tsdbDestroyCommitQueue(); -int tsdbSyncCommit(TSDB_REPO_T *repo); +int tsdbSyncCommit(STsdbRepo *repo); void tsdbIncCommitRef(int vgId); void tsdbDecCommitRef(int vgId); +// For TSDB file sync +int tsdbSyncSend(void *pRepo, SOCKET socketFd); +int tsdbSyncRecv(void *pRepo, SOCKET socketFd); + #ifdef __cplusplus } #endif diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 1303195ef149a98b6ddd7ffe55615d27eaeba04d..379c877b266b6026ea9d9ee55f76ecb24fca1a44 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -56,16 +56,6 @@ typedef struct { int32_t role[TAOS_SYNC_MAX_REPLICA]; } SNodesRole; -/* - if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file - is found between index and eindex, index shall be updated, name shall be set, size shall be set to - file size, and file magic number shall be returned. - - if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return - zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. -*/ -typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); - // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); @@ -83,24 +73,31 @@ typedef void (*FNotifyRole)(int32_t vgId, int8_t role); typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); // when data file is synced successfully, notity app -typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); +typedef void (*FStartSyncFile)(int32_t vgId); +typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); // get file version typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); +typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); +typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); + typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version SSyncCfg syncCfg; // configuration from mgmt char path[TSDB_FILENAME_LEN]; // path to the file - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; + void * pTsdb; + FGetWalInfo getWalInfoFp; + FWriteToCache writeToCacheFp; FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; - FNotifyFileSynced notifyFileSynced; - FGetVersion getVersion; + FNotifyRole notifyRoleFp; + FNotifyFlowCtrl notifyFlowCtrlFp; + FStartSyncFile startSyncFileFp; + FStopSyncFile stopSyncFileFp; + FGetVersion getVersionFp; + FSendFile sendFileFp; + FRecvFile recvFileFp; } SSyncInfo; typedef void *tsync_h; @@ -119,11 +116,6 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *); extern char *syncRole[]; //global configurable parameters -extern int32_t tsMaxSyncNum; -extern int32_t tsSyncTcpThreads; -extern int32_t tsMaxWatchFiles; -extern int32_t tsSyncTimer; -extern int32_t tsMaxFwdInfo; extern int32_t sDebugFlag; extern char tsArbitrator[]; extern uint16_t tsSyncPort; diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 0a5a3d2fa402c3dd03f1feef8cf05e446922972b..8bb9cde935cdad39aa7318517487a46a108aaa23 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_TTOKENDEF_H #define TDENGINE_TTOKENDEF_H + #define TK_ID 1 #define TK_BOOL 2 #define TK_TINYINT 3 @@ -82,146 +83,154 @@ #define TK_STABLES 64 #define TK_VGROUPS 65 #define TK_DROP 66 -#define TK_DNODE 67 -#define TK_USER 68 -#define TK_ACCOUNT 69 -#define TK_USE 70 -#define TK_DESCRIBE 71 -#define TK_ALTER 72 -#define TK_PASS 73 -#define TK_PRIVILEGE 74 -#define TK_LOCAL 75 -#define TK_IF 76 -#define TK_EXISTS 77 -#define TK_PPS 78 -#define TK_TSERIES 79 -#define TK_DBS 80 -#define TK_STORAGE 81 -#define TK_QTIME 82 -#define TK_CONNS 83 -#define TK_STATE 84 -#define TK_KEEP 85 -#define TK_CACHE 86 -#define TK_REPLICA 87 -#define TK_QUORUM 88 -#define TK_DAYS 89 -#define TK_MINROWS 90 -#define TK_MAXROWS 91 -#define TK_BLOCKS 92 -#define TK_CTIME 93 -#define TK_WAL 94 -#define TK_FSYNC 95 -#define TK_COMP 96 -#define TK_PRECISION 97 -#define TK_UPDATE 98 -#define TK_LP 99 -#define TK_RP 100 -#define TK_TAGS 101 -#define TK_USING 102 -#define TK_AS 103 -#define TK_COMMA 104 -#define TK_NULL 105 -#define TK_SELECT 106 -#define TK_UNION 107 -#define TK_ALL 108 -#define TK_FROM 109 -#define TK_VARIABLE 110 -#define TK_INTERVAL 111 -#define TK_FILL 112 -#define TK_SLIDING 113 -#define TK_ORDER 114 -#define TK_BY 115 -#define TK_ASC 116 -#define TK_DESC 117 -#define TK_GROUP 118 -#define TK_HAVING 119 -#define TK_LIMIT 120 -#define TK_OFFSET 121 -#define TK_SLIMIT 122 -#define TK_SOFFSET 123 -#define TK_WHERE 124 -#define TK_NOW 125 -#define TK_RESET 126 -#define TK_QUERY 127 -#define TK_ADD 128 -#define TK_COLUMN 129 -#define TK_TAG 130 -#define TK_CHANGE 131 -#define TK_SET 132 -#define TK_KILL 133 -#define TK_CONNECTION 134 -#define TK_STREAM 135 -#define TK_COLON 136 -#define TK_ABORT 137 -#define TK_AFTER 138 -#define TK_ATTACH 139 -#define TK_BEFORE 140 -#define TK_BEGIN 141 -#define TK_CASCADE 142 -#define TK_CLUSTER 143 -#define TK_CONFLICT 144 -#define TK_COPY 145 -#define TK_DEFERRED 146 -#define TK_DELIMITERS 147 -#define TK_DETACH 148 -#define TK_EACH 149 -#define TK_END 150 -#define TK_EXPLAIN 151 -#define TK_FAIL 152 -#define TK_FOR 153 -#define TK_IGNORE 154 -#define TK_IMMEDIATE 155 -#define TK_INITIALLY 156 -#define TK_INSTEAD 157 -#define TK_MATCH 158 -#define TK_KEY 159 -#define TK_OF 160 -#define TK_RAISE 161 -#define TK_REPLACE 162 -#define TK_RESTRICT 163 -#define TK_ROW 164 -#define TK_STATEMENT 165 -#define TK_TRIGGER 166 -#define TK_VIEW 167 -#define TK_COUNT 168 -#define TK_SUM 169 -#define TK_AVG 170 -#define TK_MIN 171 -#define TK_MAX 172 -#define TK_FIRST 173 -#define TK_LAST 174 -#define TK_TOP 175 -#define TK_BOTTOM 176 -#define TK_STDDEV 177 -#define TK_PERCENTILE 178 -#define TK_APERCENTILE 179 -#define TK_LEASTSQUARES 180 -#define TK_HISTOGRAM 181 -#define TK_DIFF 182 -#define TK_SPREAD 183 -#define TK_TWA 184 -#define TK_INTERP 185 -#define TK_LAST_ROW 186 -#define TK_RATE 187 -#define TK_IRATE 188 -#define TK_SUM_RATE 189 -#define TK_SUM_IRATE 190 -#define TK_AVG_RATE 191 -#define TK_AVG_IRATE 192 -#define TK_TBID 193 -#define TK_SEMI 194 -#define TK_NONE 195 -#define TK_PREV 196 -#define TK_LINEAR 197 -#define TK_IMPORT 198 -#define TK_METRIC 199 -#define TK_TBNAME 200 -#define TK_JOIN 201 -#define TK_METRICS 202 -#define TK_STABLE 203 -#define TK_INSERT 204 -#define TK_INTO 205 -#define TK_VALUES 206 +#define TK_STABLE 67 +#define TK_DNODE 68 +#define TK_USER 69 +#define TK_ACCOUNT 70 +#define TK_USE 71 +#define TK_DESCRIBE 72 +#define TK_ALTER 73 +#define TK_PASS 74 +#define TK_PRIVILEGE 75 +#define TK_LOCAL 76 +#define TK_IF 77 +#define TK_EXISTS 78 +#define TK_PPS 79 +#define TK_TSERIES 80 +#define TK_DBS 81 +#define TK_STORAGE 82 +#define TK_QTIME 83 +#define TK_CONNS 84 +#define TK_STATE 85 +#define TK_KEEP 86 +#define TK_CACHE 87 +#define TK_REPLICA 88 +#define TK_QUORUM 89 +#define TK_DAYS 90 +#define TK_MINROWS 91 +#define TK_MAXROWS 92 +#define TK_BLOCKS 93 +#define TK_CTIME 94 +#define TK_WAL 95 +#define TK_FSYNC 96 +#define TK_COMP 97 +#define TK_PRECISION 98 +#define TK_UPDATE 99 +#define TK_CACHELAST 100 +#define TK_LP 101 +#define TK_RP 102 +#define TK_UNSIGNED 103 +#define TK_TAGS 104 +#define TK_USING 105 +#define TK_COMMA 106 +#define TK_AS 107 +#define TK_NULL 108 +#define TK_SELECT 109 +#define TK_UNION 110 +#define TK_ALL 111 +#define TK_DISTINCT 112 +#define TK_FROM 113 +#define TK_VARIABLE 114 +#define TK_INTERVAL 115 +#define TK_FILL 116 +#define TK_SLIDING 117 +#define TK_ORDER 118 +#define TK_BY 119 +#define TK_ASC 120 +#define TK_DESC 121 +#define TK_GROUP 122 +#define TK_HAVING 123 +#define TK_LIMIT 124 +#define TK_OFFSET 125 +#define TK_SLIMIT 126 +#define TK_SOFFSET 127 +#define TK_WHERE 128 +#define TK_NOW 129 +#define TK_RESET 130 +#define TK_QUERY 131 +#define TK_ADD 132 +#define TK_COLUMN 133 +#define TK_TAG 134 +#define TK_CHANGE 135 +#define TK_SET 136 +#define TK_KILL 137 +#define TK_CONNECTION 138 +#define TK_STREAM 139 +#define TK_COLON 140 +#define TK_ABORT 141 +#define TK_AFTER 142 +#define TK_ATTACH 143 +#define TK_BEFORE 144 +#define TK_BEGIN 145 +#define TK_CASCADE 146 +#define TK_CLUSTER 147 +#define TK_CONFLICT 148 +#define TK_COPY 149 +#define TK_DEFERRED 150 +#define TK_DELIMITERS 151 +#define TK_DETACH 152 +#define TK_EACH 153 +#define TK_END 154 +#define TK_EXPLAIN 155 +#define TK_FAIL 156 +#define TK_FOR 157 +#define TK_IGNORE 158 +#define TK_IMMEDIATE 159 +#define TK_INITIALLY 160 +#define TK_INSTEAD 161 +#define TK_MATCH 162 +#define TK_KEY 163 +#define TK_OF 164 +#define TK_RAISE 165 +#define TK_REPLACE 166 +#define TK_RESTRICT 167 +#define TK_ROW 168 +#define TK_STATEMENT 169 +#define TK_TRIGGER 170 +#define TK_VIEW 171 +#define TK_COUNT 172 +#define TK_SUM 173 +#define TK_AVG 174 +#define TK_MIN 175 +#define TK_MAX 176 +#define TK_FIRST 177 +#define TK_LAST 178 +#define TK_TOP 179 +#define TK_BOTTOM 180 +#define TK_STDDEV 181 +#define TK_PERCENTILE 182 +#define TK_APERCENTILE 183 +#define TK_LEASTSQUARES 184 +#define TK_HISTOGRAM 185 +#define TK_DIFF 186 +#define TK_SPREAD 187 +#define TK_TWA 188 +#define TK_INTERP 189 +#define TK_LAST_ROW 190 +#define TK_RATE 191 +#define TK_IRATE 192 +#define TK_SUM_RATE 193 +#define TK_SUM_IRATE 194 +#define TK_AVG_RATE 195 +#define TK_AVG_IRATE 196 +#define TK_TBID 197 +#define TK_SEMI 198 +#define TK_NONE 199 +#define TK_PREV 200 +#define TK_LINEAR 201 +#define TK_IMPORT 202 +#define TK_METRIC 203 +#define TK_TBNAME 204 +#define TK_JOIN 205 +#define TK_METRICS 206 +#define TK_INSERT 207 +#define TK_INTO 208 +#define TK_VALUES 209 + + + + + #define TK_SPACE 300 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 7d5779c43f7c05ed7675cb59ef3036c14f852938..2f6b80f89d024e3d286ae20b41f645a073ddc174 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -7,27 +7,181 @@ extern "C" { #include "taosdef.h" +// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; + +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) +#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) + +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_ARRAY (1000) + #define GET_TYPED_DATA(_v, _finalType, _type, _data) \ - switch (_type) { \ - case TSDB_DATA_TYPE_TINYINT: \ - (_v) = (_finalType)GET_INT8_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_SMALLINT: \ - (_v) = (_finalType)GET_INT16_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_BIGINT: \ - (_v) = (_finalType)(GET_INT64_VAL(_data)); \ - break; \ - case TSDB_DATA_TYPE_FLOAT: \ - (_v) = (_finalType)GET_FLOAT_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_DOUBLE: \ - (_v) = (_finalType)GET_DOUBLE_VAL(_data); \ - break; \ - default: \ - (_v) = (_finalType)GET_INT32_VAL(_data); \ - break; \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + (_v) = (_finalType)GET_INT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + (_v) = (_finalType)GET_UINT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + (_v) = (_finalType)GET_INT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + (_v) = (_finalType)GET_UINT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_TIMESTAMP:\ + case TSDB_DATA_TYPE_BIGINT: \ + (_v) = (_finalType)(GET_INT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + (_v) = (_finalType)(GET_UINT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + (_v) = (_finalType)GET_FLOAT_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + (_v) = (_finalType)GET_DOUBLE_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + (_v) = (_finalType)GET_UINT32_VAL(_data); \ + break; \ + default: \ + (_v) = (_finalType)GET_INT32_VAL(_data); \ + break; \ + } \ + } while (0) + +#define SET_TYPED_DATA(_v, _type, _data) \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + *(int8_t *)(_v) = (int8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + *(uint8_t *)(_v) = (uint8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + *(int16_t *)(_v) = (int16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + *(uint16_t *)(_v) = (uint16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_BIGINT: \ + *(int64_t *)(_v) = (int64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + *(uint64_t *)(_v) = (uint64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + *(float *)(_v) = (float)(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + *(double *)(_v) = (double)(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + *(uint32_t *)(_v) = (uint32_t)(_data); \ + break; \ + default: \ + *(int32_t *)(_v) = (int32_t)(_data); \ + break; \ + } \ + } while (0) + +#define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) +#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) +#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) + +#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) + +#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) + +static FORCE_INLINE bool isNull(const char *val, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; + case TSDB_DATA_TYPE_TINYINT: + return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; + case TSDB_DATA_TYPE_SMALLINT: + return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; + case TSDB_DATA_TYPE_INT: + return *(uint32_t *)val == TSDB_DATA_INT_NULL; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; + case TSDB_DATA_TYPE_FLOAT: + return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; + case TSDB_DATA_TYPE_DOUBLE: + return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; + case TSDB_DATA_TYPE_NCHAR: + return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; + case TSDB_DATA_TYPE_BINARY: + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; + case TSDB_DATA_TYPE_UTINYINT: + return *(uint8_t*) val == TSDB_DATA_UTINYINT_NULL; + case TSDB_DATA_TYPE_USMALLINT: + return *(uint16_t*) val == TSDB_DATA_USMALLINT_NULL; + case TSDB_DATA_TYPE_UINT: + return *(uint32_t*) val == TSDB_DATA_UINT_NULL; + case TSDB_DATA_TYPE_UBIGINT: + return *(uint64_t*) val == TSDB_DATA_UBIGINT_NULL; + + default: + return false; }; +} + +typedef struct tDataTypeDescriptor { + int16_t type; + int16_t nameLen; + int32_t bytes; + char * name; + int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize); + int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize); + void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minindex, int16_t *maxindex, int16_t *numofnull); +} tDataTypeDescriptor; + +extern tDataTypeDescriptor tDataTypes[15]; + +bool isValidDataType(int32_t type); + +void setVardataNull(char* val, int32_t type); +void setNull(char *val, int32_t type, int32_t bytes); +void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); +void* getNullValue(int32_t type); + +void assignVal(char *val, const char *src, int32_t len, int32_t type); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); + +int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned); + +#define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) #ifdef __cplusplus } diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 5f643295d6e3d70999de55da47470ad0f0c0df93..7c9ebd8a0b9d96090134bd23c2061c857dfcd5f1 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -19,18 +19,9 @@ #ifdef __cplusplus extern "C" { #endif - #include "trpc.h" #include "twal.h" -typedef enum _VN_STATUS { - TAOS_VN_STATUS_INIT = 0, - TAOS_VN_STATUS_READY = 1, - TAOS_VN_STATUS_CLOSING = 2, - TAOS_VN_STATUS_UPDATING = 3, - TAOS_VN_STATUS_RESET = 4, -} EVnodeStatus; - typedef struct { int32_t len; void * rsp; @@ -43,6 +34,7 @@ typedef struct { void * rpcHandle; void * rpcAhandle; void * qhandle; + void * pVnode; int8_t qtype; int8_t msgType; SRspRet rspRet; @@ -56,33 +48,39 @@ typedef struct { void * pVnode; SRpcMsg rpcMsg; SRspRet rspRet; - char reserveForSync[16]; - SWalHead pHead[]; + char reserveForSync[24]; + SWalHead pHead; } SVWriteMsg; +// vnodeStatus extern char *vnodeStatus[]; +// vnodeMain int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); -int32_t vnodeOpen(int32_t vgId, char *rootDir); +int32_t vnodeOpen(int32_t vgId); int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeClose(int32_t vgId); -void* vnodeAcquire(int32_t vgId); // add refcount -void vnodeRelease(void *pVnode); // dec refCount +// vnodeMgmt +int32_t vnodeInitMgmt(); +void vnodeCleanupMgmt(); +void* vnodeAcquire(int32_t vgId); +void vnodeRelease(void *pVnode); void* vnodeGetWal(void *pVnode); +int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); +void vnodeBuildStatusMsg(void *pStatus); +void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); +// vnodeWrite int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg); void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite); int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet); -int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); -void vnodeBuildStatusMsg(void *pStatus); -void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code); -void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); -int32_t vnodeInitResources(); -void vnodeCleanupResources(); +// vnodeSync +void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code); +// vnodeRead int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam); void vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead); int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); @@ -91,4 +89,4 @@ int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); } #endif -#endif +#endif \ No newline at end of file diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index c86cac281c0283aac691555927d679b0ef4f4f64..b6babc5bc53aa254e0372dbbfd235bdd4cef878a 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -9,14 +9,14 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(./src SRC) LIST(REMOVE_ITEM SRC ./src/shellWindows.c) LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) - ADD_EXECUTABLE(shell ${SRC}) - + ADD_EXECUTABLE(shell ${SRC}) + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(shell taos_static) ELSE () TARGET_LINK_LIBRARIES(shell taos) ENDIF () - + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS) LIST(APPEND SRC ./src/shellEngine.c) @@ -27,7 +27,7 @@ ELSEIF (TD_WINDOWS) IF (TD_POWER) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) - ELSE () + ELSE () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () ELSEIF (TD_DARWIN) @@ -37,7 +37,10 @@ ELSEIF (TD_DARWIN) LIST(APPEND SRC ./src/shellCommand.c) LIST(APPEND SRC ./src/shellImport.c) ADD_EXECUTABLE(shell ${SRC}) - TARGET_LINK_LIBRARIES(shell taos_static) + # linking with dylib + TARGET_LINK_LIBRARIES(shell taos) + # linking taos statically + # TARGET_LINK_LIBRARIES(shell taos_static) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index d65c943e28d7d5a63aba4fc1839a4ba9cf744746..50bceb1a7169373d1c832cadf03d6d601cace264 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -30,6 +30,8 @@ #define MAX_COMMAND_SIZE 65536 #define HISTORY_FILE ".taos_history" +#define DEFAULT_RES_SHOW_NUM 100 + typedef struct SShellHistory { char* hist[MAX_HISTORY_SIZE]; int hstart; @@ -45,13 +47,13 @@ typedef struct SShellArguments { char* timezone; bool is_raw_time; bool is_use_passwd; + bool dump_config; char file[TSDB_FILENAME_LEN]; char dir[TSDB_FILENAME_LEN]; int threadNum; char* commands; int abort; int port; - int endPort; int pktLen; char* netTestRole; } SShellArguments; @@ -62,7 +64,7 @@ extern TAOS* shellInit(SShellArguments* args); extern void* shellLoopQuery(void* arg); extern void taos_error(TAOS_RES* tres, int64_t st); extern int regex_match(const char* s, const char* reg, int cflags); -void shellReadCommand(TAOS* con, char command[]); +int32_t shellReadCommand(TAOS* con, char command[]); int32_t shellRunCommand(TAOS* con, char* command); void shellRunCommandOnServer(TAOS* con, char command[]); void read_history(); @@ -86,6 +88,6 @@ extern void set_terminal_mode(); extern int get_old_terminal_mode(struct termios* tio); extern void reset_terminal_mode(); extern SShellArguments args; -extern TAOS_RES* result; +extern int64_t result; #endif diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index ddf7b21bef354817f67ccda4106abf1bc648ad6b..31ad7046e9176221e10c79b3f2367ea464529438 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -21,6 +21,8 @@ #include "shellCommand.h" #include "tkey.h" +#include "tscLog.h" + #define OPT_ABORT 1 /* �Cabort */ int indicator = 1; @@ -178,7 +180,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } -void shellReadCommand(TAOS *con, char *command) { +int32_t shellReadCommand(TAOS *con, char *command) { unsigned hist_counter = history.hend; char utf8_array[10] = "\0"; Command cmd; @@ -231,7 +233,7 @@ void shellReadCommand(TAOS *con, char *command) { sprintf(command, "%s%s", cmd.buffer, cmd.command); tfree(cmd.buffer); tfree(cmd.command); - return; + return 0; } else { updateBuffer(&cmd); } @@ -322,6 +324,8 @@ void shellReadCommand(TAOS *con, char *command) { insertChar(&cmd, &c, 1); } } + + return 0; } void *shellLoopQuery(void *arg) { @@ -340,14 +344,22 @@ void *shellLoopQuery(void *arg) { return NULL; } + int32_t err = 0; + do { // Read command from shell. memset(command, 0, MAX_COMMAND_SIZE); set_terminal_mode(); - shellReadCommand(con, command); + err = shellReadCommand(con, command); + if (err) { + break; + } reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); + tfree(command); + exitShell(); + pthread_cleanup_pop(1); return NULL; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 22f01ac142c72603b9fa0595ce8c1fc89c626d7e..a6f5869936beb7e5d9a17d0ff9ecc9fbd499db2f 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -32,21 +32,21 @@ /**************** Global variables ****************/ #ifdef _TD_POWER_ char CLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n" - "Copyright (c) 2017 by PowerDB, Inc. All rights reserved.\n\n"; + "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n"; char PROMPT_HEADER[] = "power> "; char CONTINUE_PROMPT[] = " -> "; int prompt_size = 7; #else char CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.\n\n"; + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; char PROMPT_HEADER[] = "taos> "; char CONTINUE_PROMPT[] = " -> "; int prompt_size = 6; #endif -TAOS_RES *result = NULL; +int64_t result = 0; SShellHistory history; #define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 @@ -76,7 +76,11 @@ TAOS *shellInit(SShellArguments *args) { args->user = TSDB_DEFAULT_USER; } - taos_init(); + if (taos_init()) { + printf("failed to init taos\n"); + fflush(stdout); + return NULL; + } // Connect to the database. TAOS *con = NULL; @@ -260,6 +264,14 @@ int32_t shellRunCommand(TAOS* con, char* command) { } +void freeResultWithRid(int64_t rid) { + SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); + if(pSql){ + taos_free_result(pSql); + taosReleaseRef(tscObjRef, rid); + } +} + void shellRunCommandOnServer(TAOS *con, char command[]) { int64_t st, et; wordexp_t full_path; @@ -300,21 +312,24 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { return; } + int64_t oresult = atomic_load_64(&result); + if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - atomic_store_ptr(&result, 0); - taos_free_result(pSql); + atomic_store_64(&result, 0); + freeResultWithRid(oresult); return; } if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands - int error_no = 0; + int error_no = 0; + int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - atomic_store_ptr(&result, 0); - taos_free_result(pSql); + atomic_store_64(&result, 0); + freeResultWithRid(oresult); return; } @@ -327,7 +342,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } else { int num_rows_affacted = taos_affected_rows(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected (%.6fs)\n", num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); } printf("\n"); @@ -336,8 +351,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { wordfree(&full_path); } - atomic_store_ptr(&result, 0); - taos_free_result(pSql); + atomic_store_64(&result, 0); + freeResultWithRid(oresult); } /* Function to do regular expression check */ @@ -377,10 +392,13 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } time_t tt; + int32_t ms = 0; if (precision == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)(val / 1000000); + ms = val % 1000000; } else { tt = (time_t)(val / 1000); + ms = val % 1000; } /* comment out as it make testcases like select_with_tags.sim fail. @@ -394,14 +412,22 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { #ifdef WINDOWS if (tt < 0) tt = 0; #endif + if (tt < 0 && ms != 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } struct tm* ptm = localtime(&tt); size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm); if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", (int)(val % 1000000)); + sprintf(buf + pos, ".%06d", ms); } else { - sprintf(buf + pos, ".%03d", (int)(val % 1000)); + sprintf(buf + pos, ".%03d", ms); } return buf; @@ -460,7 +486,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { wordexp_t full_path; - if (wordexp(fname, &full_path, 0) != 0) { + if (wordexp((char *)fname, &full_path, 0) != 0) { fprintf(stderr, "ERROR: invalid file name: %s\n", fname); return -1; } @@ -501,7 +527,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { row = taos_fetch_row(tres); } while( row != NULL); - result = NULL; + result = 0; fclose(fp); return numOfRows; @@ -509,7 +535,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { static void shellPrintNChar(const char *str, int length, int width) { - int pos = 0, cols = 0; + wchar_t tail[3]; + int pos = 0, cols = 0, totalCols = 0, tailLen = 0; + while (pos < length) { wchar_t wc; int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); @@ -526,15 +554,44 @@ static void shellPrintNChar(const char *str, int length, int width) { #else int w = wcwidth(wc); #endif - if (w > 0) { - if (width > 0 && cols + w > width) { - break; - } + if (w <= 0) { + continue; + } + + if (width <= 0) { + printf("%lc", wc); + continue; + } + + totalCols += w; + if (totalCols > width) { + break; + } + if (totalCols <= (width - 3)) { printf("%lc", wc); cols += w; + } else { + tail[tailLen] = wc; + tailLen++; } } + if (totalCols > width) { + // width could be 1 or 2, so printf("...") cannot be used + for (int i = 0; i < 3; i++) { + if (cols >= width) { + break; + } + putchar('.'); + ++cols; + } + } else { + for (int i = 0; i < tailLen; i++) { + printf("%lc", tail[i]); + } + cols = totalCols; + } + for (; cols < width; cols++) { putchar(' '); } @@ -562,15 +619,27 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le case TSDB_DATA_TYPE_TINYINT: printf("%*d", width, *((int8_t *)val)); break; + case TSDB_DATA_TYPE_UTINYINT: + printf("%*u", width, *((uint8_t *)val)); + break; case TSDB_DATA_TYPE_SMALLINT: printf("%*d", width, *((int16_t *)val)); break; + case TSDB_DATA_TYPE_USMALLINT: + printf("%*u", width, *((uint16_t *)val)); + break; case TSDB_DATA_TYPE_INT: printf("%*d", width, *((int32_t *)val)); break; + case TSDB_DATA_TYPE_UINT: + printf("%*u", width, *((uint32_t *)val)); + break; case TSDB_DATA_TYPE_BIGINT: printf("%*" PRId64, width, *((int64_t *)val)); break; + case TSDB_DATA_TYPE_UBIGINT: + printf("%*" PRIu64, width, *((uint64_t *)val)); + break; case TSDB_DATA_TYPE_FLOAT: printf("%*.5f", width, GET_FLOAT_VAL(val)); break; @@ -591,6 +660,17 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le } +bool isSelectQuery(TAOS_RES* tres) { + char *sql = tscGetSqlStr(tres); + + if (regex_match(sql, "^[\t ]*select[ \t]*", REG_EXTENDED | REG_ICASE)) { + return true; + } + + return false; +} + + static int verticalPrintResult(TAOS_RES* tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { @@ -609,18 +689,33 @@ static int verticalPrintResult(TAOS_RES* tres) { } } + uint64_t resShowMaxNum = UINT64_MAX; + + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + resShowMaxNum = DEFAULT_RES_SHOW_NUM; + } + int numOfRows = 0; - do { - printf("*************************** %d.row ***************************\n", numOfRows + 1); - int32_t* length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { - TAOS_FIELD* field = fields + i; + int showMore = 1; + do { + if (numOfRows < resShowMaxNum) { + printf("*************************** %d.row ***************************\n", numOfRows + 1); - int padding = (int)(maxColNameLen - strlen(field->name)); - printf("%*.s%s: ", padding, " ", field->name); + int32_t* length = taos_fetch_lengths(tres); - printField((const char*)row[i], field, 0, length[i], precision); - putchar('\n'); + for (int i = 0; i < num_fields; i++) { + TAOS_FIELD* field = fields + i; + + int padding = (int)(maxColNameLen - strlen(field->name)); + printf("%*.s%s: ", padding, " ", field->name); + + printField((const char*)row[i], field, 0, length[i], precision); + putchar('\n'); + } + } else if (showMore) { + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n"); + showMore = 0; } numOfRows++; @@ -638,15 +733,19 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { return MAX(5, width); // 'false' case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: return MAX(4, width); // '-127' case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: return MAX(6, width); // '-32767' case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: return MAX(11, width); // '-2147483648' case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: return MAX(21, width); // '-9223372036854775807' case TSDB_DATA_TYPE_FLOAT: @@ -656,13 +755,21 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { return MAX(25, width); case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: if (field->bytes > tsMaxBinaryDisplayWidth) { return MAX(tsMaxBinaryDisplayWidth, width); } else { return MAX(field->bytes, width); } + case TSDB_DATA_TYPE_NCHAR: { + int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; + if (bytes > tsMaxBinaryDisplayWidth) { + return MAX(tsMaxBinaryDisplayWidth, width); + } else { + return MAX(bytes, width); + } + } + case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { return MAX(14, width); @@ -715,16 +822,31 @@ static int horizontalPrintResult(TAOS_RES* tres) { printHeader(fields, width, num_fields); + uint64_t resShowMaxNum = UINT64_MAX; + + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + resShowMaxNum = DEFAULT_RES_SHOW_NUM; + } + int numOfRows = 0; + int showMore = 1; + do { int32_t* length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { - putchar(' '); - printField((const char*)row[i], fields + i, width[i], length[i], precision); - putchar(' '); - putchar('|'); + if (numOfRows < resShowMaxNum) { + for (int i = 0; i < num_fields; i++) { + putchar(' '); + printField((const char*)row[i], fields + i, width[i], length[i], precision); + putchar(' '); + putchar('|'); + } + putchar('\n'); + } else if (showMore) { + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); + showMore = 0; } - putchar('\n'); + numOfRows++; row = taos_fetch_row(tres); } while(row != NULL); diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 3226ad830a1995eef563fd630f8ccb6769f5e16a..07b21531a7f1cf9fc6346a27373fe9de8895f2b2 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -33,12 +33,13 @@ const char *argp_program_bug_address = ""; static char doc[] = ""; static char args_doc[] = ""; static struct argp_option options[] = { - {"host", 'h', "HOST", 0, "TDengine server IP address to connect. The default host is localhost."}, + {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, {"password", 'p', "PASSWORD", OPTION_ARG_OPTIONAL, "The password to use when connecting to the server."}, {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, {"user", 'A', "Auth", 0, "The user auth to use when connecting to the server."}, {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, + {"dump-config", 'C', 0, 0, "Dump configuration."}, {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, {"raw-time", 'r', 0, 0, "Output time as uint64_t."}, {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, @@ -46,8 +47,7 @@ static struct argp_option options[] = { {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is NULL, options: client|clients|server."}, - {"endport", 'e', "ENDPORT", 0, "Net test end port, default is 6042."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {0}}; @@ -97,6 +97,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; + case 'C': + arguments->dump_config = true; + break; case 's': arguments->commands = arg; break; @@ -130,20 +133,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'd': arguments->database = arg; break; - case 'n': arguments->netTestRole = arg; break; - - case 'e': - if (arg) { - arguments->endPort = atoi(arg); - } else { - fprintf(stderr, "Invalid end port\n"); - return -1; - } - break; - case 'l': if (arg) { arguments->pktLen = atoi(arg); @@ -152,7 +144,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { return -1; } break; - case OPT_ABORT: arguments->abort = 1; break; @@ -181,7 +172,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } -void shellReadCommand(TAOS *con, char *command) { +int32_t shellReadCommand(TAOS *con, char *command) { unsigned hist_counter = history.hend; char utf8_array[10] = "\0"; Command cmd; @@ -195,6 +186,10 @@ void shellReadCommand(TAOS *con, char *command) { while (1) { c = (char)getchar(); // getchar() return an 'int' value + if (c == EOF) { + return c; + } + if (c < 0) { // For UTF-8 int count = countPrefixOnes(c); utf8_array[0] = c; @@ -234,7 +229,7 @@ void shellReadCommand(TAOS *con, char *command) { sprintf(command, "%s%s", cmd.buffer, cmd.command); tfree(cmd.buffer); tfree(cmd.command); - return; + return 0; } else { updateBuffer(&cmd); } @@ -325,6 +320,8 @@ void shellReadCommand(TAOS *con, char *command) { insertChar(&cmd, &c, 1); } } + + return 0; } void *shellLoopQuery(void *arg) { @@ -342,12 +339,17 @@ void *shellLoopQuery(void *arg) { uError("failed to malloc command"); return NULL; } + + int32_t err = 0; do { // Read command from shell. memset(command, 0, MAX_COMMAND_SIZE); set_terminal_mode(); - shellReadCommand(con, command); + err = shellReadCommand(con, command); + if (err) { + break; + } reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 2083ad3e9b7d5a101f35af4fe20dcd38957d5be2..4c7e550760cecb7c045cb8c94fc431cb5f91812b 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -15,18 +15,35 @@ #include "os.h" #include "shell.h" +#include "tconfig.h" #include "tnettest.h" pthread_t pid; +static tsem_t cancelSem; + +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { + tsem_post(&cancelSem); +} + +void *cancelHandler(void *arg) { + while(1) { + if (tsem_wait(&cancelSem) != 0) { + taosMsleep(10); + continue; + } -void shellQueryInterruptHandler(int signum) { #ifdef LINUX - void* pResHandle = atomic_val_compare_exchange_64(&result, result, 0); - taos_stop_query(pResHandle); + int64_t rid = atomic_val_compare_exchange_64(&result, result, 0); + SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); + taos_stop_query(pSql); + taosReleaseRef(tscObjRef, rid); #else - printf("\nReceive ctrl+c or other signal, quit shell.\n"); - exit(0); + printf("\nReceive ctrl+c or other signal, quit shell.\n"); + exit(0); #endif + } + + return NULL; } int checkVersion() { @@ -58,11 +75,11 @@ SShellArguments args = { .timezone = NULL, .is_raw_time = false, .is_use_passwd = false, + .dump_config = false, .file = "\0", .dir = "\0", .threadNum = 5, - .commands = NULL, - .endPort = 6042, + .commands = NULL, .pktLen = 1000, .netTestRole = NULL }; @@ -79,11 +96,25 @@ int main(int argc, char* argv[]) { shellParseArgument(argc, argv, &args); + if (args.dump_config) { + taosInitGlobalCfg(); + taosReadGlobalLogCfg(); + + if (!taosReadGlobalCfg()) { + printf("TDengine read global config failed"); + exit(EXIT_FAILURE); + } + + taosDumpGlobalCfg(); + exit(0); + } + if (args.netTestRole && args.netTestRole[0] != 0) { - taos_init(); - CmdArguments cmdArgs; - memcpy(&cmdArgs, &args, sizeof(SShellArguments)); - taosNetTest(&cmdArgs); + if (taos_init()) { + printf("Failed to init taos"); + exit(EXIT_FAILURE); + } + taosNetTest(args.netTestRole, args.host, args.port, args.pktLen); exit(0); } @@ -93,13 +124,19 @@ int main(int argc, char* argv[]) { exit(EXIT_FAILURE); } + if (tsem_init(&cancelSem, 0, 0) != 0) { + printf("failed to create cancel semphore\n"); + exit(EXIT_FAILURE); + } + + pthread_t spid; + pthread_create(&spid, NULL, cancelHandler, NULL); + /* Interrupt handler. */ - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); - - act.sa_handler = shellQueryInterruptHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, shellQueryInterruptHandler); + taosSetSignal(SIGINT, shellQueryInterruptHandler); + taosSetSignal(SIGHUP, shellQueryInterruptHandler); + taosSetSignal(SIGABRT, shellQueryInterruptHandler); /* Get grant information */ shellGetGrantInfo(con); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index ce986813918249ebe501e92d3af307a67c296907..11c1ac34d8bc6058b33699c4d65d9d1224b88d31 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -19,12 +19,16 @@ extern char configDir[]; +void printVersion() { + printf("version: %s\n", version); +} + void printHelp() { char indent[10] = " "; printf("taos shell is used to test the TDengine database\n"); printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost."); + printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); printf("%s%s\n", indent, "-P"); @@ -35,6 +39,8 @@ void printHelp() { printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server."); printf("%s%s\n", indent, "-c"); printf("%s%s%s\n", indent, indent, "Configuration directory."); + printf("%s%s\n", indent, "-C"); + printf("%s%s%s\n", indent, indent, "Dump configuration."); printf("%s%s\n", indent, "-s"); printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell."); printf("%s%s\n", indent, "-r"); @@ -45,6 +51,12 @@ void printHelp() { printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server."); printf("%s%s\n", indent, "-t"); printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); + printf("%s%s\n", indent, "-V"); + printf("%s%s%s\n", indent, indent, "Print program version."); exit(EXIT_SUCCESS); } @@ -63,6 +75,9 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { // for password else if (strcmp(argv[i], "-p") == 0) { arguments->is_use_passwd = true; + if (i < argc - 1 && argv[i + 1][0] != '-') { + arguments->password = argv[++i]; + } } // for management port else if (strcmp(argv[i], "-P") == 0) { @@ -100,6 +115,8 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { fprintf(stderr, "Option -c requires an argument\n"); exit(EXIT_FAILURE); } + } else if (strcmp(argv[i], "-C") == 0) { + arguments->dump_config = true; } else if (strcmp(argv[i], "-s") == 0) { if (i < argc - 1) { arguments->commands = argv[++i]; @@ -137,10 +154,30 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } + else if (strcmp(argv[i], "-n") == 0) { + if (i < argc - 1) { + arguments->netTestRole = argv[++i]; + } else { + fprintf(stderr, "option -n requires an argument\n"); + exit(EXIT_FAILURE); + } + } + else if (strcmp(argv[i], "-l") == 0) { + if (i < argc - 1) { + arguments->pktLen = atoi(argv[++i]); + } else { + fprintf(stderr, "option -l requires an argument\n"); + exit(EXIT_FAILURE); + } + } + else if (strcmp(argv[i], "-V") == 0) { + printVersion(); + exit(EXIT_SUCCESS); + } // For temperory command TODO else if (strcmp(argv[i], "--help") == 0) { printHelp(); - exit(EXIT_FAILURE); + exit(EXIT_SUCCESS); } else { fprintf(stderr, "wrong options\n"); printHelp(); @@ -184,7 +221,7 @@ void insertChar(Command *cmd, char c) { cmd->command[cmd->cursorOffset++] = c; } -void shellReadCommand(TAOS *con, char command[]) { +int32_t shellReadCommand(TAOS *con, char command[]) { Command cmd; memset(&cmd, 0, sizeof(cmd)); cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); @@ -204,7 +241,7 @@ void shellReadCommand(TAOS *con, char command[]) { cmd.buffer = NULL; free(cmd.command); cmd.command = NULL; - return; + return 0; } else { shellPrintContinuePrompt(); updateBuffer(&cmd); @@ -214,6 +251,8 @@ void shellReadCommand(TAOS *con, char command[]) { insertChar(&cmd, c); } } + + return 0; } void *shellLoopQuery(void *arg) { @@ -221,12 +260,17 @@ void *shellLoopQuery(void *arg) { char *command = malloc(MAX_COMMAND_SIZE); if (command == NULL) return NULL; + int32_t err = 0; + do { memset(command, 0, MAX_COMMAND_SIZE); shellPrintPrompt(); // Read command from shell. - shellReadCommand(con, command); + err = shellReadCommand(con, command); + if (err) { + break; + } } while (shellRunCommand(con, command) == 0); return NULL; diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index ab02bdb64d1338ae2cf92a9784eddf9453a4e849..a75157b94c0bcabfccfb037b3375b0f1a4136b0f 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -2,19 +2,34 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(inc) IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) - + IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosdemo taos_static) + TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) ELSE () - TARGET_LINK_LIBRARIES(taosdemo taos) + TARGET_LINK_LIBRARIES(taosdemo taos cJson) ENDIF () ELSEIF (TD_WINDOWS) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) - TARGET_LINK_LIBRARIES(taosdemo taos_static) + SET_SOURCE_FILES_PROPERTIES(./taosdemo.c PROPERTIES COMPILE_FLAGS -w) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) + ELSE () + TARGET_LINK_LIBRARIES(taosdemo taos cJson}) + ENDIF () +ELSEIF (TD_DARWIN) + # missing a few dependencies, such as + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(taosdemo ${SRC}) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) + ELSE () + TARGET_LINK_LIBRARIES(taosdemo taos cJson) + ENDIF () ENDIF () + diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..5276b9fb61af9645578392a0035c8a95898c75e2 --- /dev/null +++ b/src/kit/taosdemo/insert.json @@ -0,0 +1,57 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/src/kit/taosdemo/query.json b/src/kit/taosdemo/query.json new file mode 100644 index 0000000000000000000000000000000000000000..33ac120bdae283f64c4419d2885a93d36a7c8093 --- /dev/null +++ b/src/kit/taosdemo/query.json @@ -0,0 +1,19 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "yes", + "databases": "dbx", + "specified_table_query": + {"query_interval":1, "concurrent":4, + "sqls": [{"sql": "select last_row(*) from stb where color='red'", "result": "./query_res0.txt"}, + {"sql": "select count(*) from stb_01", "result": "./query_res1.txt"}] + }, + "super_table_query": + {"stblname": "stb", "query_interval":1, "threads":4, + "sqls": [{"sql": "select last_row(*) from xxxx", "result": "./query_res2.txt"}] + } +} diff --git a/src/kit/taosdemo/subscribe.json b/src/kit/taosdemo/subscribe.json new file mode 100644 index 0000000000000000000000000000000000000000..fd33a2e2e2515ac268764c0a7f3f8356e998becd --- /dev/null +++ b/src/kit/taosdemo/subscribe.json @@ -0,0 +1,17 @@ +{ + "filetype":"subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "dbx", + "specified_table_query": + {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", + "sqls": [{"sql": "select avg(col1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] + }, + "super_table_query": + {"stblname": "stb", "threads":1, "mode":"sync", "interval":10000, "restart":"yes", "keepProgress":"yes", + "sqls": [{"sql": "select col1 from xxxx where col1 > 10;", "result": "./subscribe_res1.txt"}] + } +} diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 53e7d2398450fe22a11da1e7254e0c2ab5f02ea4..4a9038c71e2eb15a7677d29cd9d386ad678076fd 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -13,7 +13,13 @@ * along with this program. If not, see . */ + +/* + when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread. +*/ + #define _GNU_SOURCE +#define CURL_STATICLIB #ifdef LINUX #include "os.h" @@ -39,24 +45,134 @@ #include #include #include "os.h" + +#ifdef TD_WINDOWS + #pragma comment ( lib, "ws2_32.lib" ) + #pragma comment ( lib, "winmm.lib" ) + #pragma comment ( lib, "wldap32.lib" ) +#endif #endif +#include "cJSON.h" + #include "taos.h" +#include "taoserror.h" #include "tutil.h" +#define REQ_EXTRA_BUF_LEN 1024 +#define RESP_BUF_LEN 4096 + extern char configDir[]; -#define BUFFER_SIZE 65536 -#define MAX_DB_NAME_SIZE 64 -#define MAX_TB_NAME_SIZE 64 -#define MAX_DATA_SIZE 16000 -#define MAX_NUM_DATATYPE 10 -#define OPT_ABORT 1 /* –abort */ -#define STRING_LEN 60000 -#define MAX_PREPARED_RAND 1000000 +#define INSERT_JSON_NAME "insert.json" +#define QUERY_JSON_NAME "query.json" +#define SUBSCRIBE_JSON_NAME "subscribe.json" + +#define INSERT_MODE 0 +#define QUERY_MODE 1 +#define SUBSCRIBE_MODE 2 + +#define MAX_SQL_SIZE 65536 +#define BUFFER_SIZE (65536*2) +#define MAX_DB_NAME_SIZE 64 +#define MAX_TB_NAME_SIZE 64 +#define MAX_DATA_SIZE 16000 +#define MAX_NUM_DATATYPE 10 +#define OPT_ABORT 1 /* –abort */ +#define STRING_LEN 60000 +#define MAX_PREPARED_RAND 1000000 +#define MAX_FILE_NAME_LEN 256 + +#define MAX_SAMPLES_ONCE_FROM_FILE 10000 +#define MAX_NUM_DATATYPE 10 + +#define MAX_DB_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 8 +#define MAX_COLUMN_COUNT 1024 +#define MAX_TAG_COUNT 128 + +#define MAX_QUERY_SQL_COUNT 10 +#define MAX_QUERY_SQL_LENGTH 256 + +#define MAX_DATABASE_COUNT 256 + +typedef enum CREATE_SUB_TALBE_MOD_EN { + PRE_CREATE_SUBTBL, + AUTO_CREATE_SUBTBL, + NO_CREATE_SUBTBL +} CREATE_SUB_TALBE_MOD_EN; + +typedef enum TALBE_EXISTS_EN { + TBL_ALREADY_EXISTS, + TBL_NO_EXISTS, + TBL_EXISTS_BUTT +} TALBE_EXISTS_EN; + +enum MODE { + SYNC, + ASYNC, + MODE_BUT +}; + +enum QUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} ; + +enum _show_db_index { + TSDB_SHOW_DB_NAME_INDEX, + TSDB_SHOW_DB_CREATED_TIME_INDEX, + TSDB_SHOW_DB_NTABLES_INDEX, + TSDB_SHOW_DB_VGROUPS_INDEX, + TSDB_SHOW_DB_REPLICA_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_DAYS_INDEX, + TSDB_SHOW_DB_KEEP_INDEX, + TSDB_SHOW_DB_CACHE_INDEX, + TSDB_SHOW_DB_BLOCKS_INDEX, + TSDB_SHOW_DB_MINROWS_INDEX, + TSDB_SHOW_DB_MAXROWS_INDEX, + TSDB_SHOW_DB_WALLEVEL_INDEX, + TSDB_SHOW_DB_FSYNC_INDEX, + TSDB_SHOW_DB_COMP_INDEX, + TSDB_SHOW_DB_CACHELAST_INDEX, + TSDB_SHOW_DB_PRECISION_INDEX, + TSDB_SHOW_DB_UPDATE_INDEX, + TSDB_SHOW_DB_STATUS_INDEX, + TSDB_MAX_SHOW_DB +}; + +// -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- +enum _show_stables_index { + TSDB_SHOW_STABLES_NAME_INDEX, + TSDB_SHOW_STABLES_CREATED_TIME_INDEX, + TSDB_SHOW_STABLES_COLUMNS_INDEX, + TSDB_SHOW_STABLES_METRIC_INDEX, + TSDB_SHOW_STABLES_UID_INDEX, + TSDB_SHOW_STABLES_TID_INDEX, + TSDB_SHOW_STABLES_VGID_INDEX, + TSDB_MAX_SHOW_STABLES +}; +enum _describe_table_index { + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC +}; + +typedef struct { + char field[TSDB_COL_NAME_LEN + 1]; + char type[16]; + int length; + char note[128]; +} SColDes; /* Used by main to communicate with parse_opt. */ -typedef struct DemoArguments { +typedef struct SArguments_S { + char * metaFile; + int test_mode; char * host; uint16_t port; char * user; @@ -67,1027 +183,4528 @@ typedef struct DemoArguments { char * sqlFile; bool use_metric; bool insert_only; + bool answer_yes; + bool debug_print; char * output_file; int mode; char * datatype[MAX_NUM_DATATYPE + 1]; int len_of_binary; int num_of_CPR; int num_of_threads; + int insert_interval; int num_of_RPR; int num_of_tables; int num_of_DPT; int abort; - int order; - int rate; + int disorderRatio; + int disorderRange; int method_of_delete; char ** arg_list; -} SDemoArguments; - -#ifdef LINUX - /* The options we understand. */ - static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDengine. Default is localhost.", 0}, - {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 1}, - {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, - #ifdef _TD_POWER_ - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'powerdb'.", 3}, - #else - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, - #endif - {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, - {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, - {0, 's', "sql file", 0, "The select sql file.", 3}, - {0, 'M', 0, 0, "Use metric flag.", 13}, - {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 14}, - {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 6}, - {0, 'b', "type_of_cols", 0, "The data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'.", 7}, - {0, 'w', "length_of_binary", 0, "The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8", 8}, - {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 3.", 8}, - {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 9}, - {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 1000.", 10}, - {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 11}, - {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 100000.", 12}, - #ifdef _TD_POWER_ - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/power/'.", 14}, - #else - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 14}, - #endif - {0, 'x', 0, 0, "Insert only flag.", 13}, - {0, 'O', "order", 0, "Insert mode--0: In order, 1: Out of order. Default is in order.", 14}, - {0, 'R', "rate", 0, "Out of order data's rate--if order=1 Default 10, min: 0, max: 50.", 14}, - {0, 'D', "delete table", 0, "Delete data methods——0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database", 14}, - {0}}; - - /* Parse a single option. */ - static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - SDemoArguments *arguments = state->input; - wordexp_t full_path; - char **sptr; - switch (key) { - case 'h': - arguments->host = arg; - break; - case 'p': - arguments->port = atoi(arg); - break; - case 'u': - arguments->user = arg; - break; - case 'P': - arguments->password = arg; - break; - case 'o': - arguments->output_file = arg; - break; - case 's': - arguments->sqlFile = arg; - break; - case 'q': - arguments->mode = atoi(arg); - break; - case 'T': - arguments->num_of_threads = atoi(arg); - break; - case 'r': - arguments->num_of_RPR = atoi(arg); - break; - case 't': - arguments->num_of_tables = atoi(arg); - break; - case 'n': - arguments->num_of_DPT = atoi(arg); - break; - case 'd': - arguments->database = arg; - break; - case 'l': - arguments->num_of_CPR = atoi(arg); - break; - case 'b': - sptr = arguments->datatype; - if (strstr(arg, ",") == NULL) { - if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && - strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && - strcasecmp(arg, "SMALLINT") != 0 && - strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && - strcasecmp(arg, "BINARY") && strcasecmp(arg, "NCHAR")) { - argp_error(state, "Invalid data_type!"); - } - sptr[0] = arg; - } else { - int index = 0; - char *dupstr = strdup(arg); - char *running = dupstr; - char *token = strsep(&running, ","); - while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && - strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && - strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && - strcasecmp(token, "BIGINT") != 0 && - strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { - argp_error(state, "Invalid data_type!"); - } - sptr[index++] = token; - token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; - } - } - break; - case 'w': - arguments->len_of_binary = atoi(arg); - break; - case 'm': - arguments->tb_prefix = arg; - break; - case 'M': - arguments->use_metric = false; - break; - case 'x': - arguments->insert_only = false; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); - break; - case 'O': - arguments->order = atoi(arg); - if (arguments->order > 1 || arguments->order < 0) - { - arguments->order = 0; - } else if (arguments->order == 1) - { - arguments->rate = 10; - } - break; - case 'R': - arguments->rate = atoi(arg); - if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) - { - arguments->rate = 10; - } - break; - case 'a': - arguments->replica = atoi(arg); - if (arguments->replica > 3 || arguments->replica < 1) - { - arguments->replica = 1; - } - break; - case 'D': - arguments->method_of_delete = atoi(arg); - if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) - { - arguments->method_of_delete = 0; - } - break; - case OPT_ABORT: - arguments->abort = 1; - break; - case ARGP_KEY_ARG: - /*arguments->arg_list = &state->argv[state->next-1]; - state->next = state->argc;*/ - argp_usage(state); - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; - } - - static struct argp argp = {options, parse_opt, 0, 0}; - - void parse_args(int argc, char *argv[], SDemoArguments *arguments) { - argp_parse(&argp, argc, argv, 0, 0, arguments); - if (arguments->abort) { - #ifndef _ALPINE - error(10, 0, "ABORTED"); - #else - abort(); - #endif - } - } - -#else - void printHelp() { - char indent[10] = " "; - printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "host, The host to connect to TDengine. Default is localhost."); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, "port, The TCP/IP port number to use for the connection. Default is 0."); - printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "user, The user name to use when connecting to the server. Default is 'root'."); - printf("%s%s\n", indent, "-p"); - #ifdef _TD_POWER_ - printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'powerdb'."); - #else - printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); - #endif - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "database, Destination database. Default is 'test'."); - printf("%s%s\n", indent, "-a"); - printf("%s%s%s\n", indent, indent, "replica, Set the replica parameters of the database, Default 1, min: 1, max: 3."); - printf("%s%s\n", indent, "-m"); - printf("%s%s%s\n", indent, indent, "table_prefix, Table prefix name. Default is 't'."); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s\n", indent, indent, "sql file, The select sql file."); - printf("%s%s\n", indent, "-M"); - printf("%s%s%s\n", indent, indent, "meteric, Use metric flag."); - printf("%s%s\n", indent, "-o"); - printf("%s%s%s\n", indent, indent, "outputfile, Direct output to the named file. Default is './output.txt'."); - printf("%s%s\n", indent, "-q"); - printf("%s%s%s\n", indent, indent, "query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); - printf("%s%s\n", indent, "-b"); - printf("%s%s%s\n", indent, indent, "type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'."); - printf("%s%s\n", indent, "-w"); - printf("%s%s%s\n", indent, indent, "length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8"); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "num_of_cols_per_record, The number of columns per record. Default is 3."); - printf("%s%s\n", indent, "-T"); - printf("%s%s%s\n", indent, indent, "num_of_threads, The number of threads. Default is 10."); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s\n", indent, indent, "num_of_records_per_req, The number of records per request. Default is 1000."); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "num_of_tables, The number of tables. Default is 10000."); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, "num_of_records_per_table, The number of records per table. Default is 100000."); - printf("%s%s\n", indent, "-c"); - #ifdef _TD_POWER_ - printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/power/'."); - #else - printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); - #endif - printf("%s%s\n", indent, "-x"); - printf("%s%s%s\n", indent, indent, "flag, Insert only flag."); - printf("%s%s\n", indent, "-O"); - printf("%s%s%s\n", indent, indent, "order, Insert mode--0: In order, 1: Out of order. Default is in order."); - printf("%s%s\n", indent, "-R"); - printf("%s%s%s\n", indent, indent, "rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50."); - printf("%s%s\n", indent, "-D"); - printf("%s%s%s\n", indent, indent, "Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database."); - } - - void parse_args(int argc, char *argv[], SDemoArguments *arguments) { - char **sptr; - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0) { - arguments->host = argv[++i]; - } else if (strcmp(argv[i], "-p") == 0) { - arguments->port = atoi(argv[++i]); - } else if (strcmp(argv[i], "-u") == 0) { - arguments->user = argv[++i]; - } else if (strcmp(argv[i], "-P") == 0) { - arguments->password = argv[++i]; - } else if (strcmp(argv[i], "-o") == 0) { - arguments->output_file = argv[++i]; - } else if (strcmp(argv[i], "-s") == 0) { - arguments->sqlFile = argv[++i]; - } else if (strcmp(argv[i], "-q") == 0) { - arguments->mode = atoi(argv[++i]); - } else if (strcmp(argv[i], "-T") == 0) { - arguments->num_of_threads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - arguments->num_of_RPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - arguments->num_of_tables = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { - arguments->num_of_DPT = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0) { - arguments->database = argv[++i]; - } else if (strcmp(argv[i], "-l") == 0) { - arguments->num_of_CPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-b") == 0) { - sptr = arguments->datatype; - ++i; - if (strstr(argv[i], ",") == NULL) { - if (strcasecmp(argv[i], "INT") != 0 && strcasecmp(argv[i], "FLOAT") != 0 && - strcasecmp(argv[i], "TINYINT") != 0 && strcasecmp(argv[i], "BOOL") != 0 && - strcasecmp(argv[i], "SMALLINT") != 0 && - strcasecmp(argv[i], "BIGINT") != 0 && strcasecmp(argv[i], "DOUBLE") != 0 && - strcasecmp(argv[i], "BINARY") && strcasecmp(argv[i], "NCHAR")) { - fprintf(stderr, "Invalid data_type!\n"); - printHelp(); - exit(EXIT_FAILURE); - } - sptr[0] = argv[i]; - } else { - int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; - char *token = strsep(&running, ","); - while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && - strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && - strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && - strcasecmp(token, "BIGINT") != 0 && - strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { - fprintf(stderr, "Invalid data_type!\n"); - printHelp(); - exit(EXIT_FAILURE); - } - sptr[index++] = token; - token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; - } - } - } else if (strcmp(argv[i], "-w") == 0) { - arguments->len_of_binary = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - arguments->tb_prefix = argv[++i]; - } else if (strcmp(argv[i], "-M") == 0) { - arguments->use_metric = false; - } else if (strcmp(argv[i], "-x") == 0) { - arguments->insert_only = false; - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-O") == 0) { - arguments->order = atoi(argv[++i]); - if (arguments->order > 1 || arguments->order < 0) { - arguments->order = 0; - } else if (arguments->order == 1) { - arguments->rate = 10; - } - } else if (strcmp(argv[i], "-R") == 0) { - arguments->rate = atoi(argv[++i]); - if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) { - arguments->rate = 10; - } - } else if (strcmp(argv[i], "-a") == 0) { - arguments->replica = atoi(argv[++i]); - if (arguments->rate > 3 || arguments->rate < 1) { - arguments->rate = 1; - } - } else if (strcmp(argv[i], "-D") == 0) { - arguments->method_of_delete = atoi(argv[++i]); - if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) { - arguments->method_of_delete = 0; - } - } else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(EXIT_FAILURE); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); - } - } - } - -#endif +} SArguments; + +typedef struct SColumn_S { + char field[TSDB_COL_NAME_LEN + 1]; + char dataType[MAX_TB_NAME_SIZE]; + int dataLen; + char note[128]; +} StrColumn; + +typedef struct SSuperTable_S { + char sTblName[MAX_TB_NAME_SIZE+1]; + int childTblCount; + bool superTblExists; // 0: no, 1: yes + bool childTblExists; // 0: no, 1: yes + int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql + int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table + char childTblPrefix[MAX_TB_NAME_SIZE]; + char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample + char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful + + int multiThreadWriteOneTbl; // 0: no, 1: yes + int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl + int rowsPerTbl; // + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms or us by database precision + int maxSqlLen; // + + int64_t insertRows; // 0: no limit + int timeStampStep; + char startTimestamp[MAX_TB_NAME_SIZE]; // + char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFile[MAX_FILE_NAME_LEN+1]; + char tagsFile[MAX_FILE_NAME_LEN+1]; + + int columnCount; + StrColumn columns[MAX_COLUMN_COUNT]; + int tagCount; + StrColumn tags[MAX_TAG_COUNT]; + + char* childTblName; + char* colsOfCreateChildTable; + int lenOfOneRow; + int lenOfTagOfOneRow; + + char* sampleDataBuf; + int sampleDataBufSize; + //int sampleRowCount; + //int sampleUsePos; + + int tagSource; // 0: rand, 1: tag sample + char* tagDataBuf; + int tagSampleCount; + int tagUsePos; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; +} SSuperTable; -/* ******************************* Structure - * definition******************************* */ -enum MODE { - SYNC, ASYNC -}; typedef struct { + char name[TSDB_DB_NAME_LEN + 1]; + char create_time[32]; + int32_t ntables; + int32_t vgroups; + int16_t replica; + int16_t quorum; + int16_t days; + char keeplist[32]; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + int8_t cachelast; + char precision[8]; // time resolution + int8_t update; + char status[16]; +} SDbInfo; + +typedef struct SDbCfg_S { +// int maxtablesPerVnode; + int minRows; + int maxRows; + int comp; + int walLevel; + int cacheLast; + int fsync; + int replica; + int update; + int keep; + int days; + int cache; + int blocks; + int quorum; + char precision[MAX_TB_NAME_SIZE]; +} SDbCfg; + +typedef struct SDataBase_S { + char dbName[MAX_DB_NAME_SIZE]; + int drop; // 0: use exists, 1: if exists, drop then new create + SDbCfg dbCfg; + int superTblCount; + SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; +} SDataBase; + +typedef struct SDbs_S { + char cfgDir[MAX_FILE_NAME_LEN+1]; + char host[MAX_DB_NAME_SIZE]; + uint16_t port; + char user[MAX_DB_NAME_SIZE]; + char password[MAX_DB_NAME_SIZE]; + char resultFile[MAX_FILE_NAME_LEN+1]; + bool use_metric; + bool insert_only; + bool do_aggreFunc; + bool queryMode; + + int threadCount; + int threadCountByCreateTbl; + int dbCount; + SDataBase db[MAX_DB_COUNT]; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; + +} SDbs; + +typedef struct SuperQueryInfo_S { + int rate; // 0: unlimit > 0 loop/s + int concurrent; + int sqlCount; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int subscribeRestart; + int subscribeKeepProgress; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; +} SuperQueryInfo; + +typedef struct SubQueryInfo_S { + char sTblName[MAX_TB_NAME_SIZE+1]; + int rate; // 0: unlimit > 0 loop/s + int threadCnt; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int subscribeRestart; + int subscribeKeepProgress; + int childTblCount; + char childTblPrefix[MAX_TB_NAME_SIZE]; + int sqlCount; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + + char* childTblName; +} SubQueryInfo; + +typedef struct SQueryMetaInfo_S { + char cfgDir[MAX_FILE_NAME_LEN+1]; + char host[MAX_DB_NAME_SIZE]; + uint16_t port; + char user[MAX_DB_NAME_SIZE]; + char password[MAX_DB_NAME_SIZE]; + char dbName[MAX_DB_NAME_SIZE+1]; + char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful + + SuperQueryInfo superQueryInfo; + SubQueryInfo subQueryInfo; +} SQueryMetaInfo; + +typedef struct SThreadInfo_S { TAOS *taos; int threadID; - char db_name[MAX_DB_NAME_SIZE]; + char db_name[MAX_DB_NAME_SIZE+1]; char fp[4096]; - char **datatype; - int len_of_binary; char tb_prefix[MAX_TB_NAME_SIZE]; int start_table_id; int end_table_id; - int ncols_per_record; - int nrecords_per_table; - int nrecords_per_request; - int data_of_order; int data_of_rate; - int64_t start_time; - bool do_aggreFunc; - + uint64_t start_time; char* cols; - bool use_metric; + bool use_metric; + SSuperTable* superTblInfo; - tsem_t mutex_sem; - int notFinished; + // for async insert tsem_t lock_sem; - int counter; -} info; - -typedef struct { - TAOS *taos; - - char tb_name[MAX_TB_NAME_SIZE]; - int64_t timestamp; - int target; - int counter; - int nrecords_per_request; - int ncols_per_record; - char **data_type; - int len_of_binary; - int data_of_order; - int data_of_rate; - - tsem_t *mutex_sem; - int *notFinished; - tsem_t *lock_sem; -} sTable; - -/* ******************************* Global - * variables******************************* */ -char *aggreFunc[] = {"*", "count(*)", "avg(f1)", "sum(f1)", "max(f1)", "min(f1)", "first(f1)", "last(f1)"}; - - -void queryDB(TAOS *taos, char *command); + int64_t counter; + uint64_t st; + uint64_t et; + int64_t lastTs; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; + + // insert delay statistics + int64_t cntDelay; + int64_t totalDelay; + int64_t avgDelay; + int64_t maxDelay; + int64_t minDelay; + +} threadInfo; -void *readTable(void *sarg); +#ifdef WINDOWS +#include +// Some old MinGW/CYGWIN distributions don't define this: +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#endif -void *readMetric(void *sarg); +static HANDLE g_stdoutHandle; +static DWORD g_consoleMode; -void *syncWrite(void *sarg); +void setupForAnsiEscape(void) { + DWORD mode = 0; + g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); -void *deleteTable(); + if(g_stdoutHandle == INVALID_HANDLE_VALUE) { + exit(GetLastError()); + } -void *asyncWrite(void *sarg); + if(!GetConsoleMode(g_stdoutHandle, &mode)) { + exit(GetLastError()); + } -int generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary); + g_consoleMode = mode; -void rand_string(char *str, int size); + // Enable ANSI escape codes + mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; -void init_rand_data(); + if(!SetConsoleMode(g_stdoutHandle, mode)) { + exit(GetLastError()); + } +} -double getCurrentTime(); +void resetAfterAnsiEscape(void) { + // Reset colors + printf("\x1b[0m"); -void callBack(void *param, TAOS_RES *res, int code); -void multiThreadCreateTable(char* cols, bool use_metric, int threads, int ntables, char* db_name, char* tb_prefix, char *ip_addr, uint16_t port, char *user, char *pass); -void querySqlFile(TAOS* taos, char* sqlFile); + // Reset console mode + if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) { + exit(GetLastError()); + } +} +#else +void setupForAnsiEscape(void) {} -int main(int argc, char *argv[]) { - SDemoArguments arguments = { NULL, // host - 0, // port - "root", // user - #ifdef _TD_POWER_ - "powerdb", // password - #else - "taosdata", // password - #endif - "test", // database - 1, // replica - "t", // tb_prefix - NULL, - false, // use_metric - false, // insert_only - "./output.txt", // output_file - 0, // mode - { - "int", // datatype - "int", - "int", - "int", - "int", - "int", - "int", - "float" - }, - 8, // len_of_binary - 1, // num_of_CPR - 1, // num_of_connections/thread - 1, // num_of_RPR - 1, // num_of_tables - 50000, // num_of_DPT - 0, // abort - 0, // order - 0, // rate - 0, // method_of_delete - NULL // arg_list - }; - - /* Parse our arguments; every option seen by parse_opt will be - reflected in arguments. */ - // For demo use, change default values for some parameters; - arguments.num_of_tables = 10000; - arguments.num_of_CPR = 3; - arguments.num_of_threads = 10; - arguments.num_of_DPT = 100000; - arguments.num_of_RPR = 1000; - arguments.use_metric = true; - arguments.insert_only = true; - // end change - - parse_args(argc, argv, &arguments); - - enum MODE query_mode = arguments.mode; - char *ip_addr = arguments.host; - uint16_t port = arguments.port; - char *user = arguments.user; - char *pass = arguments.password; - char *db_name = arguments.database; - char *tb_prefix = arguments.tb_prefix; - int len_of_binary = arguments.len_of_binary; - int ncols_per_record = arguments.num_of_CPR; - int order = arguments.order; - int rate = arguments.rate; - int method_of_delete = arguments.method_of_delete; - int ntables = arguments.num_of_tables; - int threads = arguments.num_of_threads; - int nrecords_per_table = arguments.num_of_DPT; - int nrecords_per_request = arguments.num_of_RPR; - bool use_metric = arguments.use_metric; - bool insert_only = arguments.insert_only; - char **data_type = arguments.datatype; - int count_data_type = 0; - char dataString[STRING_LEN]; - bool do_aggreFunc = true; - int replica = arguments.replica; +void resetAfterAnsiEscape(void) { + // Reset colors + printf("\x1b[0m"); +} +#endif - if (NULL != arguments.sqlFile) { - TAOS* qtaos = taos_connect(ip_addr, user, pass, db_name, port); - querySqlFile(qtaos, arguments.sqlFile); - taos_close(qtaos); - return 0; - } - init_rand_data(); +static int createDatabases(); +static void createChildTables(); +static int queryDbExec(TAOS *taos, char *command, int type); - memset(dataString, 0, STRING_LEN); - int len = 0; +/* ************ Global variables ************ */ - if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 || strcasecmp(data_type[0], "NCHAR") == 0 ) { - do_aggreFunc = false; - } - for (; count_data_type <= MAX_NUM_DATATYPE; count_data_type++) { - if (data_type[count_data_type] == NULL) { - break; - } +int32_t randint[MAX_PREPARED_RAND]; +int64_t randbigint[MAX_PREPARED_RAND]; +float randfloat[MAX_PREPARED_RAND]; +double randdouble[MAX_PREPARED_RAND]; +char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", + "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; + +SArguments g_args = { + NULL, // metaFile + 0, // test_mode + "127.0.0.1", // host + 6030, // port + "root", // user + #ifdef _TD_POWER_ + "powerdb", // password + #else + "taosdata", // password + #endif + "test", // database + 1, // replica + "t", // tb_prefix + NULL, // sqlFile + false, // use_metric + false, // insert_only + false, // debug_print + false, // answer_yes; + "./output.txt", // output_file + 0, // mode : sync or async + { + "TINYINT", // datatype + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "BINARY", + "NCHAR", + "BOOL", + "TIMESTAMP" + }, + 16, // len_of_binary + 10, // num_of_CPR + 10, // num_of_connections/thread + 0, // insert_interval + 100, // num_of_RPR + 10000, // num_of_tables + 10000, // num_of_DPT + 0, // abort + 0, // disorderRatio + 1000, // disorderRange + 1, // method_of_delete + NULL // arg_list +}; - len += snprintf(dataString + len, STRING_LEN - len, "%s ", data_type[count_data_type]); - } - FILE *fp = fopen(arguments.output_file, "a"); - if (NULL == fp) { - fprintf(stderr, "Failed to open %s for writing\n", arguments.output_file); - return 1; - }; - - time_t tTime = time(NULL); - struct tm tm = *localtime(&tTime); - printf("###################################################################\n"); - printf("# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port); - printf("# User: %s\n", user); - printf("# Password: %s\n", pass); - printf("# Use metric: %s\n", use_metric ? "true" : "false"); - printf("# Datatype of Columns: %s\n", dataString); - printf("# Binary Length(If applicable): %d\n", - (strncasecmp(dataString, "BINARY", 6) == 0 || strncasecmp(dataString, "NCHAR", 5) == 0) ? len_of_binary : -1); - printf("# Number of Columns per record: %d\n", ncols_per_record); - printf("# Number of Threads: %d\n", threads); - printf("# Number of Tables: %d\n", ntables); - printf("# Number of Data per Table: %d\n", nrecords_per_table); - printf("# Records/Request: %d\n", nrecords_per_request); - printf("# Database name: %s\n", db_name); - printf("# Table prefix: %s\n", tb_prefix); - if (order == 1) - { - printf("# Data order: %d\n", order); - printf("# Data out of order rate: %d\n", rate); - - } - printf("# Delete method: %d\n", method_of_delete); - printf("# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - printf("###################################################################\n\n"); - printf("Press enter key to continue"); - (void)getchar(); - - fprintf(fp, "###################################################################\n"); - fprintf(fp, "# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port); - fprintf(fp, "# User: %s\n", user); - fprintf(fp, "# Password: %s\n", pass); - fprintf(fp, "# Use metric: %s\n", use_metric ? "true" : "false"); - fprintf(fp, "# Datatype of Columns: %s\n", dataString); - fprintf(fp, "# Binary Length(If applicable): %d\n", - (strncasecmp(dataString, "BINARY", 6) == 0 || strncasecmp(dataString, "NCHAR", 5) == 0) ? len_of_binary : -1); - fprintf(fp, "# Number of Columns per record: %d\n", ncols_per_record); - fprintf(fp, "# Number of Threads: %d\n", threads); - fprintf(fp, "# Number of Tables: %d\n", ntables); - fprintf(fp, "# Number of Data per Table: %d\n", nrecords_per_table); - fprintf(fp, "# Records/Request: %d\n", nrecords_per_request); - fprintf(fp, "# Database name: %s\n", db_name); - fprintf(fp, "# Table prefix: %s\n", tb_prefix); - if (order == 1) - { - printf("# Data order: %d\n", order); - printf("# Data out of order rate: %d\n", rate); - - } - fprintf(fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - fprintf(fp, "###################################################################\n\n"); - fprintf(fp, "| WRecords | Records/Second | Requests/Second | WLatency(ms) |\n"); - - taos_init(); - TAOS *taos = taos_connect(ip_addr, user, pass, NULL, port); - if (taos == NULL) { - fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - return 1; - } - char command[BUFFER_SIZE] = "\0"; - sprintf(command, "drop database %s;", db_name); - TAOS_RES* res = taos_query(taos, command); - taos_free_result(res); +static SDbs g_Dbs; +static int g_totalChildTables = 0; +static SQueryMetaInfo g_queryInfo; +static FILE * g_fpOfInsertResult = NULL; - sprintf(command, "create database %s replica %d;", db_name, replica); - res = taos_query(taos, command); - taos_free_result(res); +#define debugPrint(fmt, ...) \ + do { if (g_args.debug_print) fprintf(stderr, fmt, __VA_ARGS__); } while(0) +/////////////////////////////////////////////////// - char cols[STRING_LEN] = "\0"; - int colIndex = 0; - len = 0; +void printHelp() { + char indent[10] = " "; + printf("%s%s%s%s\n", indent, "-f", indent, + "The meta file to the execution procedure. Default is './meta.json'."); + printf("%s%s%s%s\n", indent, "-u", indent, + "The TDengine user name to use when connecting to the server. Default is 'root'."); +#ifdef _TD_POWER_ + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'powerdb'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/power/'."); +#else + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'taosdata'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/taos/'."); +#endif + printf("%s%s%s%s\n", indent, "-h", indent, + "The host to connect to TDengine. Default is localhost."); + printf("%s%s%s%s\n", indent, "-p", indent, + "The TCP/IP port number to use for the connection. Default is 0."); + printf("%s%s%s%s\n", indent, "-d", indent, + "Destination database. Default is 'test'."); + printf("%s%s%s%s\n", indent, "-a", indent, + "Set the replica parameters of the database, Default 1, min: 1, max: 3."); + printf("%s%s%s%s\n", indent, "-m", indent, + "Table prefix name. Default is 't'."); + printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file."); + printf("%s%s%s%s\n", indent, "-M", indent, "Use metric flag."); + printf("%s%s%s%s\n", indent, "-o", indent, + "Direct output to the named file. Default is './output.txt'."); + printf("%s%s%s%s\n", indent, "-q", indent, + "Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + printf("%s%s%s%s\n", indent, "-b", indent, + "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP."); + printf("%s%s%s%s\n", indent, "-w", indent, + "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); + printf("%s%s%s%s\n", indent, "-l", indent, + "The number of columns per record. Default is 10."); + printf("%s%s%s%s\n", indent, "-T", indent, + "The number of threads. Default is 10."); + printf("%s%s%s%s\n", indent, "-i", indent, + "The sleep time (ms) between insertion. Default is 0."); + printf("%s%s%s%s\n", indent, "-r", indent, + "The number of records per request. Default is 100."); + printf("%s%s%s%s\n", indent, "-t", indent, + "The number of tables. Default is 10000."); + printf("%s%s%s%s\n", indent, "-n", indent, + "The number of records per table. Default is 10000."); + printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); + printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); + printf("%s%s%s%s\n", indent, "-O", indent, + "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); + printf("%s%s%s%s\n", indent, "-R", indent, + "Out of order data's range, ms, default is 1000."); + printf("%s%s%s%s\n", indent, "-g", indent, + "Print debug info."); +/* printf("%s%s%s%s\n", indent, "-D", indent, + "if elete database if exists. 0: no, 1: yes, default is 1"); + */ +} - for (; colIndex < ncols_per_record - 1; colIndex++) { - if (strcasecmp(data_type[colIndex % count_data_type], "BINARY") != 0 && strcasecmp(data_type[colIndex % count_data_type], "NCHAR") != 0) { - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s", colIndex + 1, data_type[colIndex % count_data_type]); +void parse_args(int argc, char *argv[], SArguments *arguments) { + char **sptr; + wordexp_t full_path; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-f") == 0) { + arguments->metaFile = argv[++i]; + } else if (strcmp(argv[i], "-c") == 0) { + char *configPath = argv[++i]; + if (wordexp(configPath, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", configPath); + return; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + } else if (strcmp(argv[i], "-h") == 0) { + arguments->host = argv[++i]; + } else if (strcmp(argv[i], "-p") == 0) { + arguments->port = atoi(argv[++i]); + } else if (strcmp(argv[i], "-u") == 0) { + arguments->user = argv[++i]; + } else if (strcmp(argv[i], "-P") == 0) { + arguments->password = argv[++i]; + } else if (strcmp(argv[i], "-o") == 0) { + arguments->output_file = argv[++i]; + } else if (strcmp(argv[i], "-s") == 0) { + arguments->sqlFile = argv[++i]; + } else if (strcmp(argv[i], "-q") == 0) { + arguments->mode = atoi(argv[++i]); + } else if (strcmp(argv[i], "-T") == 0) { + arguments->num_of_threads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + arguments->insert_interval = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + arguments->num_of_RPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + arguments->num_of_tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + arguments->num_of_DPT = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0) { + arguments->database = argv[++i]; + } else if (strcmp(argv[i], "-l") == 0) { + arguments->num_of_CPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + sptr = arguments->datatype; + ++i; + if (strstr(argv[i], ",") == NULL) { + // only one col + if (strcasecmp(argv[i], "INT") + && strcasecmp(argv[i], "FLOAT") + && strcasecmp(argv[i], "TINYINT") + && strcasecmp(argv[i], "BOOL") + && strcasecmp(argv[i], "SMALLINT") + && strcasecmp(argv[i], "BIGINT") + && strcasecmp(argv[i], "DOUBLE") + && strcasecmp(argv[i], "BINARY") + && strcasecmp(argv[i], "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); + } + sptr[0] = argv[i]; + } else { + // more than one col + int index = 0; + char *dupstr = strdup(argv[i]); + char *running = dupstr; + char *token = strsep(&running, ","); + while (token != NULL) { + if (strcasecmp(token, "INT") + && strcasecmp(token, "FLOAT") + && strcasecmp(token, "TINYINT") + && strcasecmp(token, "BOOL") + && strcasecmp(token, "SMALLINT") + && strcasecmp(token, "BIGINT") + && strcasecmp(token, "DOUBLE") + && strcasecmp(token, "BINARY") + && strcasecmp(token, "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); + } + sptr[index++] = token; + token = strsep(&running, ","); + if (index >= MAX_NUM_DATATYPE) break; + } + sptr[index] = NULL; + } + } else if (strcmp(argv[i], "-w") == 0) { + arguments->len_of_binary = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + arguments->tb_prefix = argv[++i]; + } else if (strcmp(argv[i], "-M") == 0) { + arguments->use_metric = true; + } else if (strcmp(argv[i], "-x") == 0) { + arguments->insert_only = true; + } else if (strcmp(argv[i], "-y") == 0) { + arguments->answer_yes = true; + } else if (strcmp(argv[i], "-g") == 0) { + arguments->debug_print = true; + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-O") == 0) { + arguments->disorderRatio = atoi(argv[++i]); + if (arguments->disorderRatio > 1 + || arguments->disorderRatio < 0) { + arguments->disorderRatio = 0; + } else if (arguments->disorderRatio == 1) { + arguments->disorderRange = 10; + } + } else if (strcmp(argv[i], "-R") == 0) { + arguments->disorderRange = atoi(argv[++i]); + if (arguments->disorderRange == 1 + && (arguments->disorderRange > 50 + || arguments->disorderRange <= 0)) { + arguments->disorderRange = 10; + } + } else if (strcmp(argv[i], "-a") == 0) { + arguments->replica = atoi(argv[++i]); + if (arguments->replica > 3 || arguments->replica < 1) { + arguments->replica = 1; + } + } else if (strcmp(argv[i], "-D") == 0) { + arguments->method_of_delete = atoi(argv[++i]); + if (arguments->method_of_delete < 0 + || arguments->method_of_delete > 3) { + arguments->method_of_delete = 0; + } + } else if (strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); } else { - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s(%d)", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary); + fprintf(stderr, "wrong options\n"); + printHelp(); + exit(EXIT_FAILURE); } } - if (strcasecmp(data_type[colIndex % count_data_type], "BINARY") != 0 && strcasecmp(data_type[colIndex % count_data_type], "NCHAR") != 0){ - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s", colIndex + 1, data_type[colIndex % count_data_type]); - } else { - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s(%d)", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary); + if (arguments->debug_print) { + printf("###################################################################\n"); + printf("# meta file: %s\n", arguments->metaFile); + printf("# Server IP: %s:%hu\n", + arguments->host == NULL ? "localhost" : arguments->host, + arguments->port ); + printf("# User: %s\n", arguments->user); + printf("# Password: %s\n", arguments->password); + printf("# Use metric: %s\n", arguments->use_metric ? "true" : "false"); + if (*(arguments->datatype)) { + printf("# Specified data type: "); + for (int i = 0; i < MAX_NUM_DATATYPE; i++) + if (arguments->datatype[i]) + printf("%s,", arguments->datatype[i]); + else + break; + printf("\n"); + } + printf("# Insertion interval: %d\n", arguments->insert_interval); + printf("# Number of Columns per record: %d\n", arguments->num_of_RPR); + printf("# Number of Threads: %d\n", arguments->num_of_threads); + printf("# Number of Tables: %d\n", arguments->num_of_tables); + printf("# Number of Data per Table: %d\n", arguments->num_of_DPT); + printf("# Database name: %s\n", arguments->database); + printf("# Table prefix: %s\n", arguments->tb_prefix); + if (arguments->disorderRatio) { + printf("# Data order: %d\n", arguments->disorderRatio); + printf("# Data out of order rate: %d\n", arguments->disorderRange); + + } + printf("# Delete method: %d\n", arguments->method_of_delete); + printf("# Answer yes when prompt: %d\n", arguments->answer_yes); + printf("# Print debug info: %d\n", arguments->debug_print); + printf("###################################################################\n"); + if (!arguments->answer_yes) { + printf("Press enter key to continue\n\n"); + (void) getchar(); + } } +} - if (use_metric) { - /* Create metric table */ - printf("Creating meters super table...\n"); - snprintf(command, BUFFER_SIZE, "create table if not exists %s.meters (ts timestamp%s) tags (areaid int, loc binary(10))", db_name, cols); - queryDB(taos, command); - printf("meters created!\n"); +static bool getInfoFromJsonFile(char* file); +//static int generateOneRowDataForStb(SSuperTable* stbInfo); +//static int getDataIntoMemForStb(SSuperTable* stbInfo); +static void init_rand_data(); +void tmfclose(FILE *fp) { + if (NULL != fp) { + fclose(fp); } - taos_close(taos); - - /* Wait for table to create */ - multiThreadCreateTable(cols, use_metric, threads, ntables, db_name, tb_prefix, ip_addr, port, user, pass); - - /* Insert data */ - double ts = getCurrentTime(); - printf("Inserting data......\n"); - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - info *infos = malloc(threads * sizeof(info)); +} - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; +void tmfree(char *buf) { + if (NULL != buf) { + free(buf); } +} - int b = 0; - if (threads != 0) - b = ntables % threads; - int last = 0; - for (int i = 0; i < threads; i++) { - info *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - tstrncpy(t_info->tb_prefix, tb_prefix, MAX_TB_NAME_SIZE); - t_info->datatype = data_type; - t_info->ncols_per_record = ncols_per_record; - t_info->nrecords_per_table = nrecords_per_table; - t_info->start_time = 1500000000000; - t_info->taos = taos_connect(ip_addr, user, pass, db_name, port); - t_info->len_of_binary = len_of_binary; - t_info->nrecords_per_request = nrecords_per_request; - t_info->start_table_id = last; - t_info->data_of_order = order; - t_info->data_of_rate = rate; - t_info->end_table_id = i < b ? last + a : last + a - 1; - last = t_info->end_table_id + 1; - t_info->counter = 0; - - tsem_init(&(t_info->mutex_sem), 0, 1); - t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; - tsem_init(&(t_info->lock_sem), 0, 0); +static int queryDbExec(TAOS *taos, char *command, int type) { + int i; + TAOS_RES *res = NULL; + int32_t code = -1; - if (query_mode == SYNC) { - pthread_create(pids + i, NULL, syncWrite, t_info); - } else { - pthread_create(pids + i, NULL, asyncWrite, t_info); + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; } - } - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } - double t = getCurrentTime() - ts; - if (query_mode == SYNC) { - printf("SYNC Insert with %d connections:\n", threads); - } else { - printf("ASYNC Insert with %d connections:\n", threads); + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } } - fprintf(fp, "|%"PRIu64" | %10.2f | %10.2f | %10.4f |\n\n", - (int64_t)ntables * nrecords_per_table, ntables * nrecords_per_table / t, - ((int64_t)ntables * nrecords_per_table) / (t * nrecords_per_request), - t * 1000); - - printf("Spent %.4f seconds to insert %"PRIu64" records with %d record(s) per request: %.2f records/second\n", - t, (int64_t)ntables * nrecords_per_table, nrecords_per_request, - (int64_t)ntables * nrecords_per_table / t); - - for (int i = 0; i < threads; i++) { - info *t_info = infos + i; - taos_close(t_info->taos); - tsem_destroy(&(t_info->mutex_sem)); - tsem_destroy(&(t_info->lock_sem)); + if (code != 0) { + debugPrint("DEBUG %s() LN%d - command: %s\n", __func__, __LINE__, command); + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; } - free(pids); - free(infos); - fclose(fp); - - if (method_of_delete != 0) - { - TAOS *dtaos = taos_connect(ip_addr, user, pass, db_name, port); - double dts = getCurrentTime(); - printf("Deleteing %d table(s)......\n", ntables); + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} - switch (method_of_delete) - { - case 1: - // delete by table - /* Create all the tables; */ - for (int i = 0; i < ntables; i++) { - sprintf(command, "drop table %s.%s%d;", db_name, tb_prefix, i); - queryDB(dtaos, command); - } - break; - case 2: - // delete by stable - if (!use_metric) { - break; - } - else - { - sprintf(command, "drop table %s.meters;", db_name); - queryDB(dtaos, command); - } - break; - case 3: - // delete by database - sprintf(command, "drop database %s;", db_name); - queryDB(dtaos, command); - break; - default: - break; +static void getResult(TAOS_RES *res, char* resultFileName) { + TAOS_ROW row = NULL; + int num_rows = 0; + int num_fields = taos_field_count(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + + FILE *fp = NULL; + if (resultFileName[0] != 0) { + fp = fopen(resultFileName, "at"); + if (fp == NULL) { + fprintf(stderr, "failed to open result file: %s, result will not save to file\n", resultFileName); } - - printf("Table(s) droped!\n"); - taos_close(dtaos); - - double dt = getCurrentTime() - dts; - printf("Spent %.4f seconds to drop %d tables\n", dt, ntables); - - FILE *fp = fopen(arguments.output_file, "a"); - fprintf(fp, "Spent %.4f seconds to drop %d tables\n", dt, ntables); - fclose(fp); - } + char* databuf = (char*) calloc(1, 100*1024*1024); + if (databuf == NULL) { + fprintf(stderr, "failed to malloc, warning: save result to file slowly!\n"); + if (fp) + fclose(fp); + return ; + } - if (!insert_only) { - // query data - pthread_t read_id; - info *rInfo = malloc(sizeof(info)); - rInfo->start_time = 1500000000000; - rInfo->start_table_id = 0; - rInfo->end_table_id = ntables - 1; - rInfo->do_aggreFunc = do_aggreFunc; - rInfo->nrecords_per_table = nrecords_per_table; - rInfo->taos = taos_connect(ip_addr, user, pass, db_name, port); - strcpy(rInfo->tb_prefix, tb_prefix); - strcpy(rInfo->fp, arguments.output_file); + int totalLen = 0; + char temp[16000]; - if (!use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); - } else { - pthread_create(&read_id, NULL, readMetric, rInfo); + // fetch the records row by row + while ((row = taos_fetch_row(res))) { + if (totalLen >= 100*1024*1024 - 32000) { + if (fp) fprintf(fp, "%s", databuf); + totalLen = 0; + memset(databuf, 0, 100*1024*1024); } - pthread_join(read_id, NULL); - taos_close(rInfo->taos); + num_rows++; + int len = taos_print_row(temp, row, fields, num_fields); + len += sprintf(temp + len, "\n"); + //printf("query result:%s\n", temp); + memcpy(databuf + totalLen, temp, len); + totalLen += len; } - taos_cleanup(); - return 0; + if (fp) fprintf(fp, "%s", databuf); + tmfclose(fp); + free(databuf); } -#define MAX_SQL_SIZE 65536 -void selectSql(TAOS* taos, char* sqlcmd) -{ - TAOS_RES *pSql = taos_query(taos, sqlcmd); - int32_t code = taos_errno(pSql); - - if (code != 0) { - printf("Failed to sqlcmd:%s, reason:%s\n", sqlcmd, taos_errstr(pSql)); - taos_free_result(pSql); - exit(1); - } - - int count = 0; - while (taos_fetch_row(pSql) != NULL) { - count++; +static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { + TAOS_RES *res = taos_query(taos, command); + if (res == NULL || taos_errno(res) != 0) { + printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res)); + taos_free_result(res); + return; } - taos_free_result(pSql); - return; + getResult(res, resultFileName); + taos_free_result(res); } +double getCurrentTime() { + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { + perror("Failed to get current time in ms"); + return 0.0; + } -/* Function to do regular expression check */ -static int regexMatch(const char *s, const char *reg, int cflags) { - regex_t regex; - char msgbuf[100] = {0}; + return tv.tv_sec + tv.tv_usec / 1E6; +} - /* Compile regular expression */ - if (regcomp(®ex, reg, cflags) != 0) { - printf("Fail to compile regex\n"); - exit(-1); - } +static int32_t rand_bool(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 2; +} - /* Execute regular expression */ - int reti = regexec(®ex, s, 0, NULL, 0); - if (!reti) { - regfree(®ex); - return 1; - } else if (reti == REG_NOMATCH) { - regfree(®ex); - return 0; - } else { - regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); - printf("Regex match failed: %s\n", msgbuf); - regfree(®ex); - exit(-1); +static int32_t rand_tinyint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 128; +} + +static int32_t rand_smallint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 32767; +} + +static int32_t rand_int(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor]; +} + +static int64_t rand_bigint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randbigint[cursor]; + +} + +static float rand_float(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randfloat[cursor]; +} + +static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; +void rand_string(char *str, int size) { + str[0] = 0; + if (size > 0) { + //--size; + int n; + for (n = 0; n < size; n++) { + int key = rand_tinyint() % (int)(sizeof(charset) - 1); + str[n] = charset[key]; + } + str[n] = 0; + } +} + +static double rand_double() { + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randdouble[cursor]; + +} + +static void init_rand_data() { + for (int i = 0; i < MAX_PREPARED_RAND; i++){ + randint[i] = (int)(rand() % 65535); + randbigint[i] = (int64_t)(rand() % 2147483648); + randfloat[i] = (float)(rand() / 1000.0); + randdouble[i] = (double)(rand() / 1000000.0); } +} + +#define SHOW_PARSE_RESULT_START() \ + do { if (g_args.metaFile) \ + printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_END() \ + do { if (g_args.metaFile) \ + printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_START_TO_FILE(fp) \ + do { if (g_args.metaFile) \ + fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \ + do { if (g_args.metaFile) \ + fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \ + g_args.metaFile); } while(0) + +static int printfInsertMeta() { + SHOW_PARSE_RESULT_START(); + + printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("user: \033[33m%s\033[0m\n", g_Dbs.user); + printf("password: \033[33m%s\033[0m\n", g_Dbs.password); + printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); + printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); + printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + printf("insert interval: \033[33m%d\033[0m\n", g_args.insert_interval); + printf("number of records per req: \033[33m%d\033[0m\n", g_args.num_of_RPR); + + printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + printf("database[\033[33m%d\033[0m]:\n", i); + printf(" database name: \033[33m%s\033[0m\n", g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + printf(" drop: \033[33mno\033[0m\n"); + }else { + printf(" drop: \033[33myes\033[0m\n"); + } + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); + } else { + printf("\033[1m\033[40;31m precision error: %s\033[0m\n", + g_Dbs.db[i].dbCfg.precision); + return -1; + } + } + + printf(" super table count: \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + printf(" super table[\033[33m%d\033[0m]:\n", j); + + printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" childTblExists: \033[33m%s\033[0m\n", "error"); + } + + printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); + printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); + }else { + printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); + } + printf(" numberOfTblInOneSql: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); + printf(" rowsPerTbl: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); + printf(" disorderRange: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRange); + printf(" disorderRatio: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRatio); + printf(" maxSqlLen: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + printf(" timeStampStep: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].timeStampStep); + printf(" startTimestamp: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].startTimestamp); + printf(" sampleFormat: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFormat); + printf(" sampleFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFile); + printf(" tagsFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].tagsFile); + + printf(" columnCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + printf("column[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + printf("\n"); + + printf(" tagCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + printf("tag[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + printf("\n"); + } + printf("\n"); + } + + SHOW_PARSE_RESULT_END(); return 0; } -static int isCommentLine(char *line) { - if (line == NULL) return 1; +static void printfInsertMetaToFile(FILE* fp) { + SHOW_PARSE_RESULT_START_TO_FILE(fp); + + fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); + fprintf(fp, "user: %s\n", g_Dbs.user); + fprintf(fp, "password: %s\n", g_Dbs.password); + fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); + fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); + fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); + + fprintf(fp, "database count: %d\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + fprintf(fp, "database[%d]:\n", i); + fprintf(fp, " database name: %s\n", g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + fprintf(fp, " drop: no\n"); + }else { + fprintf(fp, " drop: yes\n"); + } - return regexMatch(line, "^\\s*#.*", REG_EXTENDED); + if (g_Dbs.db[i].dbCfg.blocks > 0) { + fprintf(fp, " blocks: %d\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + fprintf(fp, " cache: %d\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + fprintf(fp, " days: %d\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + fprintf(fp, " keep: %d\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + fprintf(fp, " replica: %d\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + fprintf(fp, " update: %d\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + fprintf(fp, " minRows: %d\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + fprintf(fp, " maxRows: %d\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + fprintf(fp, " comp: %d\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + fprintf(fp, " walLevel: %d\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + fprintf(fp, " fsync: %d\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + fprintf(fp, " quorum: %d\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + } else { + fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + } + } + + fprintf(fp, " super table count: %d\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + fprintf(fp, " super table[%d]:\n", j); + + fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "yes"); + } else { + fprintf(fp, " autoCreateTable: %s\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "yes"); + } else { + fprintf(fp, " childTblExists: %s\n", "error"); + } + + fprintf(fp, " childTblCount: %d\n", g_Dbs.db[i].superTbls[j].childTblCount); + fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); + fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); + fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + fprintf(fp, " multiThreadWriteOneTbl: no\n"); + }else { + fprintf(fp, " multiThreadWriteOneTbl: yes\n"); + } + fprintf(fp, " numberOfTblInOneSql: %d\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); + fprintf(fp, " rowsPerTbl: %d\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); + fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); + fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); + fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + + fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + fprintf(fp, "\n"); + + fprintf(fp, " tagCount: %d\n ", g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + fprintf(fp, "\n"); + } + fprintf(fp, "\n"); + } + SHOW_PARSE_RESULT_END_TO_FILE(fp); } -void querySqlFile(TAOS* taos, char* sqlFile) -{ - FILE *fp = fopen(sqlFile, "r"); +static void printfQueryMeta() { + SHOW_PARSE_RESULT_START(); + printf("host: \033[33m%s:%u\033[0m\n", g_queryInfo.host, g_queryInfo.port); + printf("user: \033[33m%s\033[0m\n", g_queryInfo.user); + printf("password: \033[33m%s\033[0m\n", g_queryInfo.password); + printf("database name: \033[33m%s\033[0m\n", g_queryInfo.dbName); + + printf("\n"); + printf("specified table query info: \n"); + printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent); + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); + + if (SUBSCRIBE_MODE == g_args.test_mode) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); + } + printf("\n"); + printf("super table query info: \n"); + printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.rate); + printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.threadCnt); + printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount); + printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.sTblName); + + if (SUBSCRIBE_MODE == g_args.test_mode) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeKeepProgress); + } + + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.sqlCount); + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]); + } + printf("\n"); + + SHOW_PARSE_RESULT_END(); +} + + +static char* xFormatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + } else { + tt = (time_t)(val / 1000); + } + +/* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", (int)(val % 1000000)); + } else { + sprintf(buf + pos, ".%03d", (int)(val % 1000)); + } + + return buf; +} + +static void xDumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) { + if (val == NULL) { + fprintf(fp, "%s", TSDB_DATA_NULL_STR); + return; + } + + char buf[TSDB_MAX_BYTES_PER_ROW]; + switch (field->type) { + case TSDB_DATA_TYPE_BOOL: + fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + fprintf(fp, "%d", *((int8_t *)val)); + break; + case TSDB_DATA_TYPE_SMALLINT: + fprintf(fp, "%d", *((int16_t *)val)); + break; + case TSDB_DATA_TYPE_INT: + fprintf(fp, "%d", *((int32_t *)val)); + break; + case TSDB_DATA_TYPE_BIGINT: + fprintf(fp, "%" PRId64, *((int64_t *)val)); + break; + case TSDB_DATA_TYPE_FLOAT: + fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + break; + case TSDB_DATA_TYPE_DOUBLE: + fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(buf, val, length); + buf[length] = 0; + fprintf(fp, "\'%s\'", buf); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + xFormatTimestamp(buf, *(int64_t*)val, precision); + fprintf(fp, "'%s'", buf); + break; + default: + break; + } +} + +static int xDumpResultToFile(const char* fname, TAOS_RES* tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + FILE* fp = fopen(fname, "at"); if (fp == NULL) { - printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); - exit(-1); + fprintf(stderr, "ERROR: failed to open file: %s\n", fname); + return -1; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + for (int col = 0; col < num_fields; col++) { + if (col > 0) { + fprintf(fp, ","); + } + fprintf(fp, "%s", fields[col].name); } + fputc('\n', fp); - int read_len = 0; - char * cmd = calloc(1, MAX_SQL_SIZE); - size_t cmd_len = 0; - char * line = NULL; - size_t line_len = 0; + int numOfRows = 0; + do { + int32_t* length = taos_fetch_lengths(tres); + for (int i = 0; i < num_fields; i++) { + if (i > 0) { + fputc(',', fp); + } + xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision); + } + fputc('\n', fp); - double t = getCurrentTime(); + numOfRows++; + row = taos_fetch_row(tres); + } while( row != NULL); + + fclose(fp); + + return numOfRows; +} + +static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; - while ((read_len = tgetline(&line, &line_len, fp)) != -1) { - if (read_len >= MAX_SQL_SIZE) continue; - line[--read_len] = '\0'; + res = taos_query(taos, "show databases;"); + int32_t code = taos_errno(res); + + if (code != 0) { + fprintf(stderr, "failed to run , reason: %s\n", taos_errstr(res)); + return -1; + } - if (read_len == 0 || isCommentLine(line)) { // line starts with # - continue; + TAOS_FIELD *fields = taos_fetch_fields(res); + + while ((row = taos_fetch_row(res)) != NULL) { + // sys database name : 'log' + if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue; + + dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); + if (dbInfos[count] == NULL) { + fprintf(stderr, "failed to allocate memory for some dbInfo[%d]\n", count); + return -1; } - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; + tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + xFormatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); + dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); + dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); + dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); + dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + + tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); + dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); + dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); + dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); + dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); + dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); + dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); + + tstrncpy(dbInfos[count]->precision, + (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); + tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX], fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); + + count++; + if (count > MAX_DATABASE_COUNT) { + fprintf(stderr, "The database count overflow than %d\n", MAX_DATABASE_COUNT); + break; } + } - memcpy(cmd + cmd_len, line, read_len); - selectSql(taos, cmd); - memset(cmd, 0, MAX_SQL_SIZE); - cmd_len = 0; + return count; +} + +static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) { + if (filename[0] == 0) + return; + + FILE *fp = fopen(filename, "at"); + if (fp == NULL) { + fprintf(stderr, "failed to open file: %s\n", filename); + return; } - t = getCurrentTime() - t; - printf("run %s took %.6f second(s)\n\n", sqlFile, t); + fprintf(fp, "================ database[%d] ================\n", index); + fprintf(fp, "name: %s\n", dbInfos->name); + fprintf(fp, "created_time: %s\n", dbInfos->create_time); + fprintf(fp, "ntables: %d\n", dbInfos->ntables); + fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); + fprintf(fp, "replica: %d\n", dbInfos->replica); + fprintf(fp, "quorum: %d\n", dbInfos->quorum); + fprintf(fp, "days: %d\n", dbInfos->days); + fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist); + fprintf(fp, "cache(MB): %d\n", dbInfos->cache); + fprintf(fp, "blocks: %d\n", dbInfos->blocks); + fprintf(fp, "minrows: %d\n", dbInfos->minrows); + fprintf(fp, "maxrows: %d\n", dbInfos->maxrows); + fprintf(fp, "wallevel: %d\n", dbInfos->wallevel); + fprintf(fp, "fsync: %d\n", dbInfos->fsync); + fprintf(fp, "comp: %d\n", dbInfos->comp); + fprintf(fp, "cachelast: %d\n", dbInfos->cachelast); + fprintf(fp, "precision: %s\n", dbInfos->precision); + fprintf(fp, "update: %d\n", dbInfos->update); + fprintf(fp, "status: %s\n", dbInfos->status); + fprintf(fp, "\n"); - free(cmd); - if (line) free(line); fclose(fp); - return; } -void * createTable(void *sarg) -{ - char command[BUFFER_SIZE] = "\0"; - - info *winfo = (info *)sarg; +static void printfQuerySystemInfo(TAOS * taos) { + char filename[MAX_QUERY_SQL_LENGTH+1] = {0}; + char buffer[MAX_QUERY_SQL_LENGTH+1] = {0}; + TAOS_RES* res; + + time_t t; + struct tm* lt; + time(&t); + lt = localtime(&t); + snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d", lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec); + + // show variables + res = taos_query(taos, "show variables;"); + //getResult(res, filename); + xDumpResultToFile(filename, res); + + // show dnodes + res = taos_query(taos, "show dnodes;"); + xDumpResultToFile(filename, res); + //getResult(res, filename); + + // show databases + res = taos_query(taos, "show databases;"); + SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *)); + if (dbInfos == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return; + } + int dbCount = getDbFromServer(taos, dbInfos); + if (dbCount <= 0) { + free(dbInfos); + return; + } + + for (int i = 0; i < dbCount; i++) { + // printf database info + printfDbInfoForQueryToFile(filename, dbInfos[i], i); + + // show db.vgroups + snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name); + res = taos_query(taos, buffer); + xDumpResultToFile(filename, res); + + // show db.stables + snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name); + res = taos_query(taos, buffer); + xDumpResultToFile(filename, res); + + free(dbInfos[i]); + } + + free(dbInfos); + +} + +void ERROR_EXIT(const char *msg) { perror(msg); exit(0); } + +int postProceSql(char* host, uint16_t port, char* sqlstr) +{ + char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\n%s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; + + char *url = "/rest/sql"; + char *auth = "Authorization: Basic cm9vdDp0YW9zZGF0YQ=="; + + struct hostent *server; + struct sockaddr_in serv_addr; + int sockfd, bytes, sent, received, req_str_len, resp_len; + char *request_buf; + char response_buf[RESP_BUF_LEN]; + uint16_t rest_port = port + TSDB_PORT_HTTP; + + int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN; + + request_buf = malloc(req_buf_len); + if (NULL == request_buf) + ERROR_EXIT("ERROR, cannot allocate memory."); + + int r = snprintf(request_buf, + req_buf_len, + req_fmt, url, host, rest_port, + auth, strlen(sqlstr), sqlstr); + if (r >= req_buf_len) { + free(request_buf); + ERROR_EXIT("ERROR too long request"); + } + printf("Request:\n%s\n", request_buf); + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + free(request_buf); + ERROR_EXIT("ERROR opening socket"); + } + + server = gethostbyname(host); + if (server == NULL) { + free(request_buf); + ERROR_EXIT("ERROR, no such host"); + } + + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons(rest_port); + memcpy(&serv_addr.sin_addr.s_addr,server->h_addr,server->h_length); + + if (connect(sockfd,(struct sockaddr *)&serv_addr,sizeof(serv_addr)) < 0) { + free(request_buf); + ERROR_EXIT("ERROR connecting"); + } + + req_str_len = strlen(request_buf); + sent = 0; + do { + bytes = write(sockfd, request_buf + sent, req_str_len - sent); + if (bytes < 0) + ERROR_EXIT("ERROR writing message to socket"); + if (bytes == 0) + break; + sent+=bytes; + } while (sent < req_str_len); + + memset(response_buf, 0, RESP_BUF_LEN); + resp_len = sizeof(response_buf) - 1; + received = 0; + do { + bytes = read(sockfd, response_buf + received, resp_len - received); + if (bytes < 0) { + free(request_buf); + ERROR_EXIT("ERROR reading response from socket"); + } + if (bytes == 0) + break; + received += bytes; + } while (received < resp_len); + + if (received == resp_len) { + free(request_buf); + ERROR_EXIT("ERROR storing complete response from socket"); + } + + response_buf[RESP_BUF_LEN - 1] = '\0'; + printf("Response:\n%s\n", response_buf); + + free(request_buf); + close(sockfd); + + return 0; +} + + +char* getTagValueFromTagSample( SSuperTable* stbInfo, int tagUsePos) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos); + + return dataBuf; +} + +char* generateTagVaulesForStb(SSuperTable* stbInfo) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "("); + for (int i = 0; i < stbInfo->tagCount; i++) { + if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", 5))) { + if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) { + printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); + tmfree(dataBuf); + return NULL; + } + + char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); + tmfree(dataBuf); + return NULL; + } + rand_string(buf, stbInfo->tags[i].dataLen); + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", 3)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", 6)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", 5)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", 6)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", 8)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", 7)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", 4)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", 4)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); + } else { + printf("No support data type: %s\n", stbInfo->tags[i].dataType); + tmfree(dataBuf); + return NULL; + } + } + dataLen -= 2; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); + return dataBuf; +} + +static int calcRowLen(SSuperTable* superTbls) { + int colIndex; + int lenOfOneRow = 0; + + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + lenOfOneRow += 21; + } else { + printf("get error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + + int tagIndex; + int lenOfTagOfOneRow = 0; + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + printf("get error tag type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + return 0; +} + + +static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, int* childTblCountOfSuperTbl) { + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + + char* childTblName = *childTblNameOfSuperTbl; + + //get all child table name use cmd: select tbname from superTblName; + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s", dbName, sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + + int childTblCount = 10000; + int count = 0; + childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); + char* pTblName = childTblName; + while ((row = taos_fetch_row(res)) != NULL) { + int32_t* len = taos_fetch_lengths(res); + tstrncpy(pTblName, (char *)row[0], len[0]+1); + //printf("==== sub table name: %s\n", pTblName); + count++; + if (count >= childTblCount - 1) { + char *tmp = realloc(childTblName, (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1); + if (tmp != NULL) { + childTblName = tmp; + childTblCount = (int)(childTblCount*1.5); + memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN)); + } else { + // exit, if allocate more memory failed + printf("realloc fail for save child table name of %s.%s\n", dbName, sTblName); + tmfree(childTblName); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + } + pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; + } + + *childTblCountOfSuperTbl = count; + *childTblNameOfSuperTbl = childTblName; + + taos_free_result(res); + return 0; +} + +static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + //get schema use cmd: describe superTblName; + snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + return -1; + } + + int tagIndex = 0; + int columnIndex = 0; + TAOS_FIELD *fields = taos_fetch_fields(res); + while ((row = taos_fetch_row(res)) != NULL) { + if (0 == count) { + count++; + continue; + } + + if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { + tstrncpy(superTbls->tags[tagIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->tags[tagIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + tagIndex++; + } else { + tstrncpy(superTbls->columns[columnIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->columns[columnIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + columnIndex++; + } + count++; + } + + superTbls->columnCount = columnIndex; + superTbls->tagCount = tagIndex; + taos_free_result(res); + + calcRowLen(superTbls); + + if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { + //get all child table name use cmd: select tbname from superTblName; + getAllChildNameOfSuperTable(taos, dbName, superTbls->sTblName, &superTbls->childTblName, &superTbls->childTblCount); + } + return 0; +} + +static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { + char command[BUFFER_SIZE] = "\0"; + + char cols[STRING_LEN] = "\0"; + int colIndex; + int len = 0; + + int lenOfOneRow = 0; + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(cols + len, STRING_LEN - len, + ", col%d %s(%d)", colIndex, "BINARY", + superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(cols + len, STRING_LEN - len, + ", col%d %s(%d)", colIndex, "NCHAR", + superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT"); + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT"); + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE"); + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP"); + lenOfOneRow += 21; + } else { + taos_close(taos); + printf("config error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); + + // save for creating child table + superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1); + if (NULL == superTbls->colsOfCreateChildTable) { + printf("Failed when calloc, size:%d", len+1); + taos_close(taos); + exit(-1); + } + snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols); + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable); + + if (use_metric) { + char tags[STRING_LEN] = "\0"; + int tagIndex; + len = 0; + + int lenOfTagOfOneRow = 0; + len += snprintf(tags + len, STRING_LEN - len, "("); + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "BINARY", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "NCHAR", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "INT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BIGINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "SMALLINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "TINYINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BOOL"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "FLOAT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "DOUBLE"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + taos_close(taos); + printf("config error tag type : %s\n", dataType); + exit(-1); + } + } + len -= 2; + len += snprintf(tags + len, STRING_LEN - len, ")"); + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + snprintf(command, BUFFER_SIZE, + "create table if not exists %s.%s (ts timestamp%s) tags %s", + dbName, superTbls->sTblName, cols, tags); + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, command); + + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + fprintf(stderr, "create supertable %s failed!\n\n", superTbls->sTblName); + return -1; + } + debugPrint("DEBUG - create supertable %s success!\n\n", superTbls->sTblName); + } + return 0; +} + + +static int createDatabases() { + TAOS * taos = NULL; + int ret = 0; + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + return -1; + } + char command[BUFFER_SIZE] = "\0"; + + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].drop) { + sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); + debugPrint("DEBUG %s() %d command: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + taos_close(taos); + return -1; + } + } + + int dataLen = 0; + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "create database if not exists %s ", g_Dbs.db[i].dbName); + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "blocks %d ", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "cache %d ", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "days %d ", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "keep %d ", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "replica %d ", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "update %d ", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, + // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "minrows %d ", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "maxrows %d ", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "comp %d ", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "wal %d ", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.cacheLast > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "cachelast %d ", g_Dbs.db[i].dbCfg.cacheLast); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "fsync %d ", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } + + debugPrint("DEBUG %s() %d command: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + taos_close(taos); + printf("\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); + + debugPrint("DEBUG %s() %d count:%d\n", __func__, __LINE__, g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // describe super table, if exists + sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); + debugPrint("DEBUG %s() %d command: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS; + ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); + } else { + g_Dbs.db[i].superTbls[j].superTblExists = TBL_ALREADY_EXISTS; + ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j]); + } + + if (0 != ret) { + printf("\ncreate super table %d failed!\n\n", j); + taos_close(taos); + return -1; + } + } + } + + taos_close(taos); + return 0; +} + +static void* createTable(void *sarg) +{ + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + int64_t lastPrintTime = taosGetTimestampMs(); + + int buff_len; + if (superTblInfo) + buff_len = superTblInfo->maxSqlLen; + else + buff_len = BUFFER_SIZE; + + char *buffer = calloc(buff_len, 1); + if (buffer == NULL) { + fprintf(stderr, "Memory allocated failed!"); + exit(-1); + } + + int len = 0; + int batchNum = 0; + //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); + for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { + if (0 == g_Dbs.use_metric) { + snprintf(buffer, buff_len, + "create table if not exists %s.%s%d %s;", + winfo->db_name, + g_args.tb_prefix, i, + winfo->cols); + } else { + if (0 == len) { + batchNum = 0; + memset(buffer, 0, buff_len); + len += snprintf(buffer + len, + buff_len - len, "create table "); + } + + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + i % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + free(buffer); + return NULL; + } + + len += snprintf(buffer + len, + superTblInfo->maxSqlLen - len, + "if not exists %s.%s%d using %s.%s tags %s ", + winfo->db_name, superTblInfo->childTblPrefix, + i, winfo->db_name, + superTblInfo->sTblName, tagsValBuf); + free(tagsValBuf); + batchNum++; + + if ((batchNum < superTblInfo->batchCreateTableNum) + && ((superTblInfo->maxSqlLen - len) + >= (superTblInfo->lenOfTagOfOneRow + 256))) { + continue; + } + } + + len = 0; + debugPrint("DEBUG %s() LN%d %s\n", __func__, __LINE__, buffer); + if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){ + free(buffer); + return NULL; + } + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] already create %d - %d tables\n", + winfo->threadID, winfo->start_table_id, i); + lastPrintTime = currentPrintTime; + } + } + + if (0 != len) { + debugPrint("DEBUG %s() %d buffer: %s\n", __func__, __LINE__, buffer); + (void)queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE); + } + + free(buffer); + return NULL; +} + +int startMultiThreadCreateChildTable( + char* cols, int threads, int ntables, + char* db_name, SSuperTable* superTblInfo) { + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed\n"); + exit(-1); + } + + if (threads < 1) { + threads = 1; + } + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + b = ntables % threads; + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->superTblInfo = superTblInfo; + t_info->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + db_name, + g_Dbs.port); + if (t_info->taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + free(pids); + free(infos); + return -1; + } + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + last = t_info->end_table_id + 1; + t_info->use_metric = 1; + t_info->cols = cols; + t_info->minDelay = INT16_MAX; + pthread_create(pids + i, NULL, createTable, t_info); + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + taos_close(t_info->taos); + } + + free(pids); + free(infos); + + return 0; +} + + +static void createChildTables() { + char tblColsBuf[MAX_SQL_SIZE]; + int len; + + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].superTblCount > 0) { + // with super table + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) + || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + startMultiThreadCreateChildTable( + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, + g_Dbs.threadCountByCreateTbl, + g_Dbs.db[i].superTbls[j].childTblCount, + g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + } + } else { + // normal table + len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); + int i = 0; + while (g_args.datatype[i]) { + if ((strncasecmp(g_args.datatype[i], "BINARY", strlen("BINARY")) == 0) + || (strncasecmp(g_args.datatype[i], "NCHAR", strlen("NCHAR")) == 0)) { + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, ", COL%d %s(60)", i, g_args.datatype[i]); + } else { + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, ", COL%d %s", i, g_args.datatype[i]); + } + len = strlen(tblColsBuf); + } + + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); + + debugPrint("DEBUG - %s() LN%d: %s\n", __func__, __LINE__, + tblColsBuf); + startMultiThreadCreateChildTable( + tblColsBuf, + g_Dbs.threadCountByCreateTbl, + g_args.num_of_DPT, + g_Dbs.db[i].dbName, + NULL); + } + } +} + +/* +static int taosGetLineNum(const char *fileName) +{ + int lineNum = 0; + char cmd[1024] = { 0 }; + char buf[1024] = { 0 }; + sprintf(cmd, "wc -l %s", fileName); + + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); + return lineNum; + } + + if (fgets(buf, sizeof(buf), fp)) { + int index = strchr((const char*)buf, ' ') - buf; + buf[index] = '\0'; + lineNum = atoi(buf); + } + pclose(fp); + return lineNum; +} +*/ + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + + FILE *fp = fopen(superTblInfo->tagsFile, "r"); + if (fp == NULL) { + printf("Failed to open tags file: %s, reason:%s\n", + superTblInfo->tagsFile, strerror(errno)); + return -1; + } + + if (superTblInfo->tagDataBuf) { + free(superTblInfo->tagDataBuf); + superTblInfo->tagDataBuf = NULL; + } + + int tagCount = 10000; + int count = 0; + char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount); + if (tagDataBuf == NULL) { + printf("Failed to calloc, reason:%s\n", strerror(errno)); + fclose(fp); + return -1; + } + + while ((readLen = tgetline(&line, &n, fp)) != -1) { + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen); + count++; + + if (count >= tagCount - 1) { + char *tmp = realloc(tagDataBuf, + (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow); + if (tmp != NULL) { + tagDataBuf = tmp; + tagCount = (int)(tagCount*1.5); + memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow, + 0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow)); + } else { + // exit, if allocate more memory failed + printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile); + tmfree(tagDataBuf); + free(line); + fclose(fp); + return -1; + } + } + } + + superTblInfo->tagDataBuf = tagDataBuf; + superTblInfo->tagSampleCount = count; + + free(line); + fclose(fp); + return 0; +} + +int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { + // TODO + return 0; +} + + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sampleBuf) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + int getRows = 0; + + memset(sampleBuf, 0, MAX_SAMPLES_ONCE_FROM_FILE* superTblInfo->lenOfOneRow); + while (1) { + readLen = tgetline(&line, &n, fp); + if (-1 == readLen) { + if(0 != fseek(fp, 0, SEEK_SET)) { + printf("Failed to fseek file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); + return -1; + } + continue; + } + + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + if (readLen > superTblInfo->lenOfOneRow) { + printf("sample row len[%d] overflow define schema len[%d], so discard this row\n", + (int32_t)readLen, superTblInfo->lenOfOneRow); + continue; + } + + memcpy(sampleBuf + getRows * superTblInfo->lenOfOneRow, line, readLen); + getRows++; + + if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { + break; + } + } + + tmfree(line); + return 0; +} + +/* +void readSampleFromFileToMem(SSuperTable * supterTblInfo) { + int ret; + if (0 == strncasecmp(supterTblInfo->sampleFormat, "csv", 3)) { + ret = readSampleFromCsvFileToMem(supterTblInfo); + } else if (0 == strncasecmp(supterTblInfo->sampleFormat, "json", 4)) { + ret = readSampleFromJsonFileToMem(supterTblInfo); + } + + if (0 != ret) { + exit(-1); + } +} +*/ +static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* superTbls) { + bool ret = false; + + // columns + cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); + if (columns && columns->type != cJSON_Array) { + printf("failed to read json, columns not found\n"); + goto PARSE_OVER; + } else if (NULL == columns) { + superTbls->columnCount = 0; + superTbls->tagCount = 0; + return true; + } + + int columnSize = cJSON_GetArraySize(columns); + if (columnSize > MAX_COLUMN_COUNT) { + printf("failed to read json, column size overflow, max column size is %d\n", + MAX_COLUMN_COUNT); + goto PARSE_OVER; + } + + int count = 1; + int index = 0; + StrColumn columnCase; + + //superTbls->columnCount = columnSize; + for (int k = 0; k < columnSize; ++k) { + cJSON* column = cJSON_GetArrayItem(columns, k); + if (column == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(column, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("failed to read json, column count not found"); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(column, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + printf("failed to read json, column type not found"); + goto PARSE_OVER; + } + //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(column, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + printf("failed to read json, column len not found"); + goto PARSE_OVER; + } else { + columnCase.dataLen = 8; + } + + for (int n = 0; n < count; ++n) { + tstrncpy(superTbls->columns[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->columns[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->columnCount = index; + + count = 1; + index = 0; + // tags + cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); + if (!tags || tags->type != cJSON_Array) { + printf("failed to read json, tags not found"); + goto PARSE_OVER; + } + + int tagSize = cJSON_GetArraySize(tags); + if (tagSize > MAX_TAG_COUNT) { + printf("failed to read json, tags size overflow, max tag size is %d\n", MAX_TAG_COUNT); + goto PARSE_OVER; + } + + //superTbls->tagCount = tagSize; + for (int k = 0; k < tagSize; ++k) { + cJSON* tag = cJSON_GetArrayItem(tags, k); + if (tag == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(tag, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("failed to read json, column count not found"); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(tag, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + printf("failed to read json, tag type not found"); + goto PARSE_OVER; + } + tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + printf("failed to read json, column len not found"); + goto PARSE_OVER; + } else { + columnCase.dataLen = 0; + } + + for (int n = 0; n < count; ++n) { + tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->tags[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->tagCount = index; + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromInsertJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + tstrncpy(g_Dbs.host, host->valuestring, MAX_DB_NAME_SIZE); + } else if (!host) { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_Dbs.port = port->valueint; + } else if (!port) { + g_Dbs.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + tstrncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE); + } else if (!user) { + tstrncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE); + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + tstrncpy(g_Dbs.password, password->valuestring, MAX_DB_NAME_SIZE); + } else if (!password) { + tstrncpy(g_Dbs.password, "taosdata", MAX_DB_NAME_SIZE); + } + + cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); + if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { + tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); + } else if (!resultfile) { + tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); + } + + cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); + if (threads && threads->type == cJSON_Number) { + g_Dbs.threadCount = threads->valueint; + } else if (!threads) { + g_Dbs.threadCount = 1; + } else { + printf("failed to read json, threads not found"); + goto PARSE_OVER; + } + + cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); + if (threads2 && threads2->type == cJSON_Number) { + g_Dbs.threadCountByCreateTbl = threads2->valueint; + } else if (!threads2) { + g_Dbs.threadCountByCreateTbl = 1; + } else { + printf("failed to read json, threads2 not found"); + goto PARSE_OVER; + } + + cJSON* insertInterval = cJSON_GetObjectItem(root, "insert_interval"); + if (insertInterval && insertInterval->type == cJSON_Number) { + g_args.insert_interval = insertInterval->valueint; + } else if (!insertInterval) { + g_args.insert_interval = 0; + } else { + printf("failed to read json, insert_interval not found"); + goto PARSE_OVER; + } + + cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); + if (numRecPerReq && numRecPerReq->type == cJSON_Number) { + g_args.num_of_RPR = numRecPerReq->valueint; + } else if (!numRecPerReq) { + g_args.num_of_RPR = 100; + } else { + printf("failed to read json, num_of_records_per_req not found"); + goto PARSE_OVER; + } + + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, + if (answerPrompt + && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { + if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { + g_args.answer_yes = false; + } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { + g_args.answer_yes = true; + } else { + g_args.answer_yes = false; + } + } else if (!answerPrompt) { + g_args.answer_yes = false; + } else { + printf("failed to read json, confirm_parameter_prompt not found"); + goto PARSE_OVER; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (!dbs || dbs->type != cJSON_Array) { + printf("failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + int dbSize = cJSON_GetArraySize(dbs); + if (dbSize > MAX_DB_COUNT) { + printf("failed to read json, databases size overflow, max database is %d\n", MAX_DB_COUNT); + goto PARSE_OVER; + } + + g_Dbs.dbCount = dbSize; + for (int i = 0; i < dbSize; ++i) { + cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); + if (dbinfos == NULL) continue; + + // dbinfo + cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); + if (!dbinfo || dbinfo->type != cJSON_Object) { + printf("failed to read json, dbinfo not found"); + goto PARSE_OVER; + } + + cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); + if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { + printf("failed to read json, db name not found"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); + + cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); + if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { + if (0 == strncasecmp(drop->valuestring, "yes", 3)) { + g_Dbs.db[i].drop = 1; + } else { + g_Dbs.db[i].drop = 0; + } + } else if (!drop) { + g_Dbs.db[i].drop = 0; + } else { + printf("failed to read json, drop not found"); + goto PARSE_OVER; + } + + cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); + if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, MAX_DB_NAME_SIZE); + } else if (!precision) { + //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, precision not found"); + goto PARSE_OVER; + } + + cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); + if (update && update->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.update = update->valueint; + } else if (!update) { + g_Dbs.db[i].dbCfg.update = -1; + } else { + printf("failed to read json, update not found"); + goto PARSE_OVER; + } + + cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); + if (replica && replica->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.replica = replica->valueint; + } else if (!replica) { + g_Dbs.db[i].dbCfg.replica = -1; + } else { + printf("failed to read json, replica not found"); + goto PARSE_OVER; + } + + cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); + if (keep && keep->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.keep = keep->valueint; + } else if (!keep) { + g_Dbs.db[i].dbCfg.keep = -1; + } else { + printf("failed to read json, keep not found"); + goto PARSE_OVER; + } + + cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); + if (days && days->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.days = days->valueint; + } else if (!days) { + g_Dbs.db[i].dbCfg.days = -1; + } else { + printf("failed to read json, days not found"); + goto PARSE_OVER; + } + + cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); + if (cache && cache->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cache = cache->valueint; + } else if (!cache) { + g_Dbs.db[i].dbCfg.cache = -1; + } else { + printf("failed to read json, cache not found"); + goto PARSE_OVER; + } + + cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); + if (blocks && blocks->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.blocks = blocks->valueint; + } else if (!blocks) { + g_Dbs.db[i].dbCfg.blocks = -1; + } else { + printf("failed to read json, block not found"); + goto PARSE_OVER; + } + + //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); + //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; + //} else if (!maxtablesPerVnode) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; + //} else { + // printf("failed to read json, maxtablesPerVnode not found"); + // goto PARSE_OVER; + //} + + cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); + if (minRows && minRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.minRows = minRows->valueint; + } else if (!minRows) { + g_Dbs.db[i].dbCfg.minRows = -1; + } else { + printf("failed to read json, minRows not found"); + goto PARSE_OVER; + } + + cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); + if (maxRows && maxRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; + } else if (!maxRows) { + g_Dbs.db[i].dbCfg.maxRows = -1; + } else { + printf("failed to read json, maxRows not found"); + goto PARSE_OVER; + } + + cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); + if (comp && comp->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.comp = comp->valueint; + } else if (!comp) { + g_Dbs.db[i].dbCfg.comp = -1; + } else { + printf("failed to read json, comp not found"); + goto PARSE_OVER; + } + + cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); + if (walLevel && walLevel->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; + } else if (!walLevel) { + g_Dbs.db[i].dbCfg.walLevel = -1; + } else { + printf("failed to read json, walLevel not found"); + goto PARSE_OVER; + } + + cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast"); + if (cacheLast && cacheLast->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint; + } else if (!cacheLast) { + g_Dbs.db[i].dbCfg.cacheLast = -1; + } else { + printf("failed to read json, cacheLast not found"); + goto PARSE_OVER; + } + + cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); + if (quorum && quorum->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.quorum = quorum->valueint; + } else if (!quorum) { + g_Dbs.db[i].dbCfg.quorum = -1; + } else { + printf("failed to read json, walLevel not found"); + goto PARSE_OVER; + } + + cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); + if (fsync && fsync->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.fsync = fsync->valueint; + } else if (!fsync) { + g_Dbs.db[i].dbCfg.fsync = -1; + } else { + printf("failed to read json, fsync not found"); + goto PARSE_OVER; + } + + // super_talbes + cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); + if (!stables || stables->type != cJSON_Array) { + printf("failed to read json, super_tables not found"); + goto PARSE_OVER; + } + + int stbSize = cJSON_GetArraySize(stables); + if (stbSize > MAX_SUPER_TABLE_COUNT) { + printf("failed to read json, databases size overflow, max database is %d\n", MAX_SUPER_TABLE_COUNT); + goto PARSE_OVER; + } + + g_Dbs.db[i].superTblCount = stbSize; + for (int j = 0; j < stbSize; ++j) { + cJSON* stbInfo = cJSON_GetArrayItem(stables, j); + if (stbInfo == NULL) continue; + + // dbinfo + cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); + if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { + printf("failed to read json, stb name not found"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); + + cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); + if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { + printf("failed to read json, childtable_prefix not found"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); + + cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null + if (autoCreateTbl + && autoCreateTbl->type == cJSON_String + && autoCreateTbl->valuestring != NULL) { + if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; + } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + } else if (!autoCreateTbl) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + printf("failed to read json, auto_create_table not found"); + goto PARSE_OVER; + } + + cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); + if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; + } else if (!batchCreateTbl) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; + } else { + printf("failed to read json, batch_create_tbl_num not found"); + goto PARSE_OVER; + } + + cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no + if (childTblExists + && childTblExists->type == cJSON_String + && childTblExists->valuestring != NULL) { + if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; + } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } + } else if (!childTblExists) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + printf("failed to read json, child_table_exists not found"); + goto PARSE_OVER; + } + + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); + if (!count || count->type != cJSON_Number || 0 >= count->valueint) { + printf("failed to read json, childtable_count not found"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; + + cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); + if (dataSource && dataSource->type == cJSON_String + && dataSource->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, + dataSource->valuestring, MAX_DB_NAME_SIZE); + } else if (!dataSource) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, data_source not found"); + goto PARSE_OVER; + } + + cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful + if (insertMode && insertMode->type == cJSON_String + && insertMode->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, + insertMode->valuestring, MAX_DB_NAME_SIZE); + } else if (!insertMode) { + tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, insert_mode not found"); + goto PARSE_OVER; + } + + cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); + if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, ts->valuestring, MAX_DB_NAME_SIZE); + } else if (!ts) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, "now", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, start_timestamp not found"); + goto PARSE_OVER; + } + + cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); + if (timestampStep && timestampStep->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; + } else if (!timestampStep) { + g_Dbs.db[i].superTbls[j].timeStampStep = 1000; + } else { + printf("failed to read json, timestamp_step not found"); + goto PARSE_OVER; + } + + cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size"); + if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint; + if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } + } else if (!sampleDataBufSize) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } else { + printf("failed to read json, sample_buf_size not found"); + goto PARSE_OVER; + } + + cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); + if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, + sampleFormat->valuestring, MAX_DB_NAME_SIZE); + } else if (!sampleFormat) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, sample_format not found"); + goto PARSE_OVER; + } + + cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); + if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, + sampleFile->valuestring, MAX_FILE_NAME_LEN); + } else if (!sampleFile) { + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, sample_file not found"); + goto PARSE_OVER; + } + + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); + if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, + tagsFile->valuestring, MAX_FILE_NAME_LEN); + if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + g_Dbs.db[i].superTbls[j].tagSource = 1; + } + } else if (!tagsFile) { + memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + printf("failed to read json, tags_file not found"); + goto PARSE_OVER; + } + + cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + int32_t len = maxSqlLen->valueint; + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < TSDB_MAX_SQL_LEN) { + len = TSDB_MAX_SQL_LEN; + } + g_Dbs.db[i].superTbls[j].maxSqlLen = len; + } else if (!maxSqlLen) { + g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; + } else { + printf("failed to read json, maxSqlLen not found"); + goto PARSE_OVER; + } + + cJSON *multiThreadWriteOneTbl = + cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes + if (multiThreadWriteOneTbl + && multiThreadWriteOneTbl->type == cJSON_String + && multiThreadWriteOneTbl->valuestring != NULL) { + if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; + } else { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } + } else if (!multiThreadWriteOneTbl) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } else { + printf("failed to read json, multiThreadWriteOneTbl not found"); + goto PARSE_OVER; + } + + cJSON* numberOfTblInOneSql = cJSON_GetObjectItem(stbInfo, "number_of_tbl_in_one_sql"); + if (numberOfTblInOneSql && numberOfTblInOneSql->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = numberOfTblInOneSql->valueint; + } else if (!numberOfTblInOneSql) { + g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = 0; + } else { + printf("failed to read json, numberOfTblInOneSql not found"); + goto PARSE_OVER; + } + + cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "rows_per_tbl"); + if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint; + } else if (!rowsPerTbl) { + g_Dbs.db[i].superTbls[j].rowsPerTbl = 1; + } else { + printf("failed to read json, rowsPerTbl not found"); + goto PARSE_OVER; + } + + cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); + if (disorderRatio && disorderRatio->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; + } else if (!disorderRatio) { + g_Dbs.db[i].superTbls[j].disorderRatio = 0; + } else { + printf("failed to read json, disorderRatio not found"); + goto PARSE_OVER; + } + + cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); + if (disorderRange && disorderRange->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; + } else if (!disorderRange) { + g_Dbs.db[i].superTbls[j].disorderRange = 1000; + } else { + printf("failed to read json, disorderRange not found"); + goto PARSE_OVER; + } + + + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); + if (insertRows && insertRows->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; + //if (0 == g_Dbs.db[i].superTbls[j].insertRows) { + // g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + //} + } else if (!insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } else { + printf("failed to read json, insert_rows not found"); + goto PARSE_OVER; + } + + if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable + || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + + int retVal = getColumnAndTagTypeFromInsertJsonFile(stbInfo, &g_Dbs.db[i].superTbls[j]); + if (false == retVal) { + goto PARSE_OVER; + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromQueryJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + tstrncpy(g_queryInfo.host, host->valuestring, MAX_DB_NAME_SIZE); + } else if (!host) { + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_queryInfo.port = port->valueint; + } else if (!port) { + g_queryInfo.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + tstrncpy(g_queryInfo.user, user->valuestring, MAX_DB_NAME_SIZE); + } else if (!user) { + tstrncpy(g_queryInfo.user, "root", MAX_DB_NAME_SIZE); ; + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + tstrncpy(g_queryInfo.password, password->valuestring, MAX_DB_NAME_SIZE); + } else if (!password) { + tstrncpy(g_queryInfo.password, "taosdata", MAX_DB_NAME_SIZE);; + } + + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, + if (answerPrompt && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { + if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { + g_args.answer_yes = false; + } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { + g_args.answer_yes = true; + } else { + g_args.answer_yes = false; + } + } else if (!answerPrompt) { + g_args.answer_yes = false; + } else { + printf("failed to read json, confirm_parameter_prompt not found"); + goto PARSE_OVER; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { + tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); + } else if (!dbs) { + printf("failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); + if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { + tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); + } else if (!queryMode) { + tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); + } else { + printf("failed to read json, query_mode not found\n"); + goto PARSE_OVER; + } + + // super_table_query + cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query"); + if (!superQuery) { + g_queryInfo.superQueryInfo.concurrent = 0; + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superQuery->type != cJSON_Object) { + printf("failed to read json, super_table_query not found"); + goto PARSE_OVER; + } else { + cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval"); + if (rate && rate->type == cJSON_Number) { + g_queryInfo.superQueryInfo.rate = rate->valueint; + } else if (!rate) { + g_queryInfo.superQueryInfo.rate = 0; + } + + cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent"); + if (concurrent && concurrent->type == cJSON_Number) { + g_queryInfo.superQueryInfo.concurrent = concurrent->valueint; + } else if (!concurrent) { + g_queryInfo.superQueryInfo.concurrent = 1; + } + + cJSON* mode = cJSON_GetObjectItem(superQuery, "mode"); + if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { + if (0 == strcmp("sync", mode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", mode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 1; + } else { + printf("failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } + + cJSON* interval = cJSON_GetObjectItem(superQuery, "interval"); + if (interval && interval->type == cJSON_Number) { + g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint; + } else if (!interval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.subscribeInterval = 10000; + } + + cJSON* restart = cJSON_GetObjectItem(superQuery, "restart"); + if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { + if (0 == strcmp("yes", restart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", restart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 0; + } else { + printf("failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } + + cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (keepProgress + && keepProgress->type == cJSON_String + && keepProgress->valuestring != NULL) { + if (0 == strcmp("yes", keepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", keepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } else { + printf("failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!superSqls) { + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superSqls->type != cJSON_Array) { + printf("failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(superSqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.superQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(superSqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("failed to read json, sql not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { + tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, super query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + // sub_table_query + cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query"); + if (!subQuery) { + g_queryInfo.subQueryInfo.threadCnt = 0; + g_queryInfo.subQueryInfo.sqlCount = 0; + } else if (subQuery->type != cJSON_Object) { + printf("failed to read json, sub_table_query not found"); + ret = true; + goto PARSE_OVER; + } else { + cJSON* subrate = cJSON_GetObjectItem(subQuery, "query_interval"); + if (subrate && subrate->type == cJSON_Number) { + g_queryInfo.subQueryInfo.rate = subrate->valueint; + } else if (!subrate) { + g_queryInfo.subQueryInfo.rate = 0; + } + + cJSON* threads = cJSON_GetObjectItem(subQuery, "threads"); + if (threads && threads->type == cJSON_Number) { + g_queryInfo.subQueryInfo.threadCnt = threads->valueint; + } else if (!threads) { + g_queryInfo.subQueryInfo.threadCnt = 1; + } + + //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count"); + //if (subTblCnt && subTblCnt->type == cJSON_Number) { + // g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint; + //} else if (!subTblCnt) { + // g_queryInfo.subQueryInfo.childTblCount = 0; + //} + + cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname"); + if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { + tstrncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); + } else { + printf("failed to read json, super table name not found\n"); + goto PARSE_OVER; + } + + cJSON* submode = cJSON_GetObjectItem(subQuery, "mode"); + if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { + if (0 == strcmp("sync", submode->valuestring)) { + g_queryInfo.subQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", submode->valuestring)) { + g_queryInfo.subQueryInfo.subscribeMode = 1; + } else { + printf("failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeMode = 0; + } + + cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval"); + if (subinterval && subinterval->type == cJSON_Number) { + g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint; + } else if (!subinterval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.subQueryInfo.subscribeInterval = 10000; + } + + cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart"); + if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { + if (0 == strcmp("yes", subrestart->valuestring)) { + g_queryInfo.subQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", subrestart->valuestring)) { + g_queryInfo.subQueryInfo.subscribeRestart = 0; + } else { + printf("failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeRestart = 1; + } + + cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress"); + if (subkeepProgress && subkeepProgress->type == cJSON_String && subkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", subkeepProgress->valuestring)) { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + } else { + printf("failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls"); + if (!subsqls) { + g_queryInfo.subQueryInfo.sqlCount = 0; + } else if (subsqls->type != cJSON_Array) { + printf("failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(subsqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.subQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(subsqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("failed to read json, sql not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ + tstrncpy(g_queryInfo.subQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, sub query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getInfoFromJsonFile(char* file) { + debugPrint("DEBUG - %s %d %s\n", __func__, __LINE__, file); + + FILE *fp = fopen(file, "r"); + if (!fp) { + printf("failed to read %s, reason:%s\n", file, strerror(errno)); + return false; + } + + bool ret = false; + int maxLen = 64000; + char *content = calloc(1, maxLen + 1); + int len = fread(content, 1, maxLen, fp); + if (len <= 0) { + free(content); + fclose(fp); + printf("failed to read %s, content is null", file); + return false; + } + + content[len] = 0; + cJSON* root = cJSON_Parse(content); + if (root == NULL) { + printf("failed to cjson parse %s, invalid json format", file); + goto PARSE_OVER; + } + + cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); + if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { + if (0 == strcasecmp("insert", filetype->valuestring)) { + g_args.test_mode = INSERT_MODE; + } else if (0 == strcasecmp("query", filetype->valuestring)) { + g_args.test_mode = QUERY_MODE; + } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { + g_args.test_mode = SUBSCRIBE_MODE; + } else { + printf("failed to read json, filetype not support\n"); + goto PARSE_OVER; + } + } else if (!filetype) { + g_args.test_mode = INSERT_MODE; + } else { + printf("failed to read json, filetype not found\n"); + goto PARSE_OVER; + } + + if (INSERT_MODE == g_args.test_mode) { + ret = getMetaFromInsertJsonFile(root); + } else if (QUERY_MODE == g_args.test_mode) { + ret = getMetaFromQueryJsonFile(root); + } else if (SUBSCRIBE_MODE == g_args.test_mode) { + ret = getMetaFromQueryJsonFile(root); + } else { + printf("input json file type error! please input correct file type: insert or query or subscribe\n"); + goto PARSE_OVER; + } + +PARSE_OVER: + free(content); + cJSON_Delete(root); + fclose(fp); + return ret; +} + +void prePareSampleData() { + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + //if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].dataSource, "sample", 6)) { + // readSampleFromFileToMem(&g_Dbs.db[i].superTbls[j]); + //} + + if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { + (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]); + } + } + } +} + +void postFreeResource() { + tmfclose(g_fpOfInsertResult); + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) { + free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { + free(g_Dbs.db[i].superTbls[j].sampleDataBuf); + g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) { + free(g_Dbs.db[i].superTbls[j].tagDataBuf); + g_Dbs.db[i].superTbls[j].tagDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].childTblName) { + free(g_Dbs.db[i].superTbls[j].childTblName); + g_Dbs.db[i].superTbls[j].childTblName = NULL; + } + } + } +} + +int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* superTblInfo, int* sampleUsePos, FILE *fp, char* sampleBuf) { + if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) { + int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleBuf); + if (0 != ret) { + return -1; + } + *sampleUsePos = 0; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%s", sampleBuf + superTblInfo->lenOfOneRow * (*sampleUsePos)); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + + (*sampleUsePos)++; + + return dataLen; +} + +int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) { + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + for (int i = 0; i < stbInfo->columnCount; i++) { + if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { + if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { + printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); + return (-1); + } + + char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", stbInfo->columns[i].dataLen); + return (-1); + } + rand_string(buf, stbInfo->columns[i].dataLen); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "int", 3)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bigint", 6)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "float", 5)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "double", 6)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "smallint", 8)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + } else { + printf("No support data type: %s\n", stbInfo->columns[i].dataType); + return (-1); + } + } + dataLen -= 2; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + + return dataLen; +} + +static void syncWriteForNumberOfTblInOneSql( + threadInfo *winfo, FILE *fp, char* sampleDataBuf) { + SSuperTable* superTblInfo = winfo->superTblInfo; + + int samplePos = 0; + + //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); + int64_t totalRowsInserted = 0; + int64_t totalAffectedRows = 0; + int64_t lastPrintTime = taosGetTimestampMs(); + + char* buffer = calloc(superTblInfo->maxSqlLen+1, 1); + if (NULL == buffer) { + printf("========calloc size[ %d ] fail!\n", superTblInfo->maxSqlLen); + return; + } + + int32_t numberOfTblInOneSql = superTblInfo->numberOfTblInOneSql; + int32_t tbls = winfo->end_table_id - winfo->start_table_id + 1; + if (numberOfTblInOneSql > tbls) { + numberOfTblInOneSql = tbls; + } + + uint64_t time_counter = winfo->start_time; + int64_t tmp_time; + int sampleUsePos; + + int64_t st = 0; + int64_t et = 0; + for (int i = 0; i < superTblInfo->insertRows;) { + int32_t tbl_id = 0; + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { + int inserted = i; + + for (int k = 0; k < g_args.num_of_RPR;) { + int len = 0; + memset(buffer, 0, superTblInfo->maxSqlLen); + char *pstr = buffer; + + int32_t end_tbl_id = tID + numberOfTblInOneSql; + if (end_tbl_id > winfo->end_table_id) { + end_tbl_id = winfo->end_table_id+1; + } + + for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) { + sampleUsePos = samplePos; + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, tbl_id % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + goto free_and_statistics; + } + + if (0 == len) { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d using %s.%s tags %s values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id, + winfo->db_name, + superTblInfo->sTblName, + tagsValBuf); + } else { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + " %s.%s%d using %s.%s tags %s values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id, + winfo->db_name, + superTblInfo->sTblName, + tagsValBuf); + } + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + if (0 == len) { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s values ", + winfo->db_name, + superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + } else { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + " %s.%s values ", + winfo->db_name, + superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + } + } else { // pre-create child table + if (0 == len) { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id); + } else { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + " %s.%s%d values ", + winfo->db_name, + superTblInfo->childTblPrefix, + tbl_id); + } + } + + tmp_time = time_counter; + for (k = 0; k < superTblInfo->rowsPerTbl;) { + int retLen = 0; + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample"))) { + retLen = getRowDataFromSample(pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo, + &sampleUsePos, + fp, + sampleDataBuf); + if (retLen < 0) { + goto free_and_statistics; + } + } else if (0 == strncasecmp( + superTblInfo->dataSource, "rand", strlen("rand"))) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio + && rand_num < superTblInfo->disorderRatio) { + int64_t d = tmp_time - rand() % superTblInfo->disorderRange; + retLen = generateRowData(pstr + len, + superTblInfo->maxSqlLen - len, + d, + superTblInfo); + } else { + retLen = generateRowData(pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo); + } + if (retLen < 0) { + goto free_and_statistics; + } + } + len += retLen; + //inserted++; + k++; + totalRowsInserted++; + + if (inserted >= superTblInfo->insertRows || + (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) { + tID = tbl_id + 1; + printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", + superTblInfo->lenOfOneRow); + goto send_to_server; + } + } + } + + tID = tbl_id; + inserted += superTblInfo->rowsPerTbl; + +send_to_server: + if (g_args.insert_interval && (g_args.insert_interval > (et - st))) { + int sleep_time = g_args.insert_interval - (et -st); + debugPrint("DEBUG sleep: %d ms\n", sleep_time); + taosMsleep(sleep_time); // ms + } + + if (g_args.insert_interval) { + st = taosGetTimestampMs(); + } + + if (0 == strncasecmp(superTblInfo->insertMode, + "taosc", + strlen("taosc"))) { + //printf("multi table===== sql: %s \n\n", buffer); + //int64_t t1 = taosGetTimestampMs(); + int64_t startTs; + int64_t endTs; + startTs = taosGetTimestampUs(); + + debugPrint("DEBUG %s() LN%d buff: %s\n", __func__, __LINE__, buffer); + int affectedRows = queryDbExec( + winfo->taos, buffer, INSERT_TYPE); + + if (0 > affectedRows) { + goto free_and_statistics; + } else { + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + if (delay > winfo->maxDelay) winfo->maxDelay = delay; + if (delay < winfo->minDelay) winfo->minDelay = delay; + winfo->cntDelay++; + winfo->totalDelay += delay; + //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; + } + totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + winfo->threadID, + totalRowsInserted, + totalAffectedRows); + lastPrintTime = currentPrintTime; + } + //int64_t t2 = taosGetTimestampMs(); + //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); + } else { + //int64_t t1 = taosGetTimestampMs(); + int retCode = postProceSql(g_Dbs.host, g_Dbs.port, buffer); + //int64_t t2 = taosGetTimestampMs(); + //printf("http insert sql return, Spent %ld ms \n", t2 - t1); + + if (0 != retCode) { + printf("========restful return fail, threadID[%d]\n", winfo->threadID); + goto free_and_statistics; + } + } + if (g_args.insert_interval) { + et = taosGetTimestampMs(); + } + + break; + } + + if (tID > winfo->end_table_id) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { + samplePos = sampleUsePos; + } + i = inserted; + time_counter = tmp_time; + } + } + + //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); + } + +free_and_statistics: + tmfree(buffer); + winfo->totalRowsInserted = totalRowsInserted; + winfo->totalAffectedRows = totalAffectedRows; + printf("====thread[%d] completed total inserted rows: %"PRId64 ", affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + return; +} + +int32_t generateData(char *res, char **data_type, + int num_of_cols, int64_t timestamp, int len_of_binary) { + memset(res, 0, MAX_DATA_SIZE); + char *pstr = res; + pstr += sprintf(pstr, "(%" PRId64, timestamp); + int c = 0; + + for (; c < MAX_NUM_DATATYPE; c++) { + if (data_type[c] == NULL) { + break; + } + } + + if (0 == c) { + perror("data type error!"); + exit(-1); + } + + for (int i = 0; i < num_of_cols; i++) { + if (strcasecmp(data_type[i % c], "tinyint") == 0) { + pstr += sprintf(pstr, ", %d", rand_tinyint() ); + } else if (strcasecmp(data_type[i % c], "smallint") == 0) { + pstr += sprintf(pstr, ", %d", rand_smallint()); + } else if (strcasecmp(data_type[i % c], "int") == 0) { + pstr += sprintf(pstr, ", %d", rand_int()); + } else if (strcasecmp(data_type[i % c], "bigint") == 0) { + pstr += sprintf(pstr, ", %" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % c], "float") == 0) { + pstr += sprintf(pstr, ", %10.4f", rand_float()); + } else if (strcasecmp(data_type[i % c], "double") == 0) { + double t = rand_double(); + pstr += sprintf(pstr, ", %20.8f", t); + } else if (strcasecmp(data_type[i % c], "bool") == 0) { + bool b = rand() & 1; + pstr += sprintf(pstr, ", %s", b ? "true" : "false"); + } else if (strcasecmp(data_type[i % c], "binary") == 0) { + char *s = malloc(len_of_binary); + rand_string(s, len_of_binary); + pstr += sprintf(pstr, ", \"%s\"", s); + free(s); + }else if (strcasecmp(data_type[i % c], "nchar") == 0) { + char *s = malloc(len_of_binary); + rand_string(s, len_of_binary); + pstr += sprintf(pstr, ", \"%s\"", s); + free(s); + } + + if (pstr - res > MAX_DATA_SIZE) { + perror("column length too long, abort"); + exit(-1); + } + } + + pstr += sprintf(pstr, ")"); + + return (int32_t)(pstr - res); +} + +// sync insertion +/* + 1 thread: 100 tables * 2000 rows/s + 1 thread: 10 tables * 20000 rows/s + 6 thread: 300 tables * 2000 rows/s + + 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s +*/ +static void* syncWrite(void *sarg) { + + threadInfo *winfo = (threadInfo *)sarg; + + char buffer[BUFFER_SIZE] = "\0"; + char data[MAX_DATA_SIZE]; + char **data_type = g_args.datatype; + int len_of_binary = g_args.len_of_binary; + + int ncols_per_record = 1; // count first col ts + int i = 0; + while(g_args.datatype[i]) { + i ++; + ncols_per_record ++; + } + + srand((uint32_t)time(NULL)); + int64_t time_counter = winfo->start_time; + + uint64_t st = 0; + uint64_t et = 0; + + for (int i = 0; i < g_args.num_of_DPT;) { + + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { + int inserted = i; + int64_t tmp_time = time_counter; + + char *pstr = buffer; + pstr += sprintf(pstr, + "insert into %s.%s%d values", + winfo->db_name, g_args.tb_prefix, tID); + int k; + for (k = 0; k < g_args.num_of_RPR;) { + int rand_num = rand() % 100; + int len = -1; + + if ((g_args.disorderRatio != 0) + && (rand_num < g_args.disorderRange)) { + + int64_t d = tmp_time - rand() % 1000000 + rand_num; + len = generateData(data, data_type, + ncols_per_record, d, len_of_binary); + } else { + len = generateData(data, data_type, + ncols_per_record, tmp_time += 1000, len_of_binary); + } + + //assert(len + pstr - buffer < BUFFER_SIZE); + if (len + pstr - buffer >= BUFFER_SIZE) { // too long + break; + } + + pstr += sprintf(pstr, " %s", data); + inserted++; + k++; + + if (inserted >= g_args.num_of_DPT) + break; + } + + /* puts(buffer); */ + int64_t startTs; + int64_t endTs; + startTs = taosGetTimestampUs(); + //queryDB(winfo->taos, buffer); + if (i > 0 && g_args.insert_interval + && (g_args.insert_interval > (et - st) )) { + int sleep_time = g_args.insert_interval - (et -st); + debugPrint("DEBUG sleep: %d ms\n", sleep_time); + taosMsleep(sleep_time); // ms + } + + if (g_args.insert_interval) { + st = taosGetTimestampMs(); + } + debugPrint("DEBUG - %s() LN%d %s\n", __func__, __LINE__, buffer); + int affectedRows = queryDbExec(winfo->taos, buffer, 1); + + if (0 <= affectedRows){ + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + if (delay > winfo->maxDelay) + winfo->maxDelay = delay; + if (delay < winfo->minDelay) + winfo->minDelay = delay; + winfo->cntDelay++; + winfo->totalDelay += delay; + //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; + } + + if (g_args.insert_interval) { + et = taosGetTimestampMs(); + } + + if (tID == winfo->end_table_id) { + i = inserted; + time_counter = tmp_time; + } + } + + } + return NULL; +} + + +static void* syncWriteWithStb(void *sarg) { + uint64_t totalRowsInserted = 0; + uint64_t totalAffectedRows = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + FILE *fp = NULL; + char* sampleDataBuf = NULL; + int samplePos = 0; + + // each thread read sample data from csv file + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", + strlen("sample"))) { + sampleDataBuf = calloc( + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); + if (sampleDataBuf == NULL) { + printf("Failed to calloc %d Bytes, reason:%s\n", + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, + strerror(errno)); + return NULL; + } + + fp = fopen(superTblInfo->sampleFile, "r"); + if (fp == NULL) { + printf("Failed to open sample file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); + tmfree(sampleDataBuf); + return NULL; + } + int ret = readSampleFromCsvFileToMem(fp, + superTblInfo, sampleDataBuf); + if (0 != ret) { + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; + } + } + + if (superTblInfo->numberOfTblInOneSql > 0) { + syncWriteForNumberOfTblInOneSql(winfo, fp, sampleDataBuf); + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; + } + + char* buffer = calloc(superTblInfo->maxSqlLen, 1); + if (NULL == buffer) { + printf("Failed to calloc %d Bytes, reason:%s\n", + superTblInfo->maxSqlLen, + strerror(errno)); + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; + } + + int64_t time_counter = winfo->start_time; + uint64_t st = 0; + uint64_t et = 0; + + debugPrint("DEBUG - %s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, superTblInfo->insertRows); + + for (int i = 0; i < superTblInfo->insertRows;) { + + for (uint32_t tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { + uint64_t inserted = i; + uint64_t tmp_time = time_counter; + + int sampleUsePos = samplePos; + int k = 0; + debugPrint("DEBUG - %s() LN%d num_of_RPR=%d\n", __func__, __LINE__, g_args.num_of_RPR); + for (k = 0; k < g_args.num_of_RPR;) { + int len = 0; + memset(buffer, 0, superTblInfo->maxSqlLen); + char *pstr = buffer; + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + tID % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + goto free_and_statistics_2; + } + + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d using %s.%s tags %s values", + winfo->db_name, + superTblInfo->childTblPrefix, + tID, + winfo->db_name, + superTblInfo->sTblName, + tagsValBuf); + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s values", + winfo->db_name, + superTblInfo->childTblName + tID * TSDB_TABLE_NAME_LEN); + } else { + len += snprintf(pstr + len, + superTblInfo->maxSqlLen - len, + "insert into %s.%s%d values", + winfo->db_name, + superTblInfo->childTblPrefix, + tID); + } + + int retLen = 0; + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { + retLen = getRowDataFromSample( + pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo, + &sampleUsePos, + fp, + sampleDataBuf); + if (retLen < 0) { + goto free_and_statistics_2; + } + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio + && rand_num < superTblInfo->disorderRatio) { + int64_t d = tmp_time - rand() % superTblInfo->disorderRange; + retLen = generateRowData( + pstr + len, + superTblInfo->maxSqlLen - len, d, + superTblInfo); + //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); + } else { + retLen = generateRowData( + pstr + len, + superTblInfo->maxSqlLen - len, + tmp_time += superTblInfo->timeStampStep, + superTblInfo); + } + if (retLen < 0) { + goto free_and_statistics_2; + } + } +/* len += retLen; +*/ + inserted++; + k++; + totalRowsInserted++; + + if (inserted > superTblInfo->insertRows) + break; +/* if (inserted >= superTblInfo->insertRows + || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) + break; +*/ + if (i > 0 && g_args.insert_interval + && (g_args.insert_interval > (et - st) )) { + int sleep_time = g_args.insert_interval - (et -st); + debugPrint("DEBUG sleep: %d ms\n", sleep_time); + taosMsleep(sleep_time); // ms + } + + if (g_args.insert_interval) { + st = taosGetTimestampMs(); + } + + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { + //printf("===== sql: %s \n\n", buffer); + //int64_t t1 = taosGetTimestampMs(); + int64_t startTs; + int64_t endTs; + startTs = taosGetTimestampUs(); + + debugPrint("DEBUG %s() LN%d %s\n", __func__, __LINE__, buffer); + int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + + if (0 > affectedRows){ + goto free_and_statistics_2; + } else { + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + if (delay > winfo->maxDelay) winfo->maxDelay = delay; + if (delay < winfo->minDelay) winfo->minDelay = delay; + winfo->cntDelay++; + winfo->totalDelay += delay; + //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; + } + totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + winfo->threadID, + totalRowsInserted, + totalAffectedRows); + lastPrintTime = currentPrintTime; + } + //int64_t t2 = taosGetTimestampMs(); + //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); + } else { + //int64_t t1 = taosGetTimestampMs(); + int retCode = postProceSql(g_Dbs.host, g_Dbs.port, buffer); + //int64_t t2 = taosGetTimestampMs(); + //printf("http insert sql return, Spent %ld ms \n", t2 - t1); + + if (0 != retCode) { + printf("========restful return fail, threadID[%d]\n", winfo->threadID); + goto free_and_statistics_2; + } + } + if (g_args.insert_interval) { + et = taosGetTimestampMs(); + } +/* + if (loop_cnt) { + loop_cnt--; + if ((1 == loop_cnt) && (0 != nrecords_last_req)) { + nrecords_cur_req = nrecords_last_req; + } else if (0 == loop_cnt){ + nrecords_cur_req = nrecords_no_last_req; + loop_cnt = loop_cnt_orig; + break; + } + } else { + break; + } + */ + } + + if (tID == winfo->end_table_id) { + if (0 == strncasecmp( + superTblInfo->dataSource, "sample", strlen("sample"))) { + samplePos = sampleUsePos; + } + + i = inserted; + time_counter = tmp_time; + } + } + + //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); + } + +free_and_statistics_2: + tmfree(buffer); + tmfree(sampleDataBuf); + tmfclose(fp); + + winfo->totalRowsInserted = totalRowsInserted; + winfo->totalAffectedRows = totalAffectedRows; + + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + winfo->threadID, + totalRowsInserted, + totalAffectedRows); + return NULL; +} + +void callBack(void *param, TAOS_RES *res, int code) { + threadInfo* winfo = (threadInfo*)param; + + if (g_args.insert_interval) { + winfo->et = taosGetTimestampMs(); + if (winfo->et - winfo->st < 1000) { + taosMsleep(1000 - (winfo->et - winfo->st)); // ms + } + } + + char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); + char *data = calloc(1, MAX_DATA_SIZE); + char *pstr = buffer; + pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id); + if (winfo->counter >= winfo->superTblInfo->insertRows) { + winfo->start_table_id++; + winfo->counter = 0; + } + if (winfo->start_table_id > winfo->end_table_id) { + tsem_post(&winfo->lock_sem); + free(buffer); + free(data); + taos_free_result(res); + return; + } + + for (int i = 0; i < g_args.num_of_RPR; i++) { + int rand_num = rand() % 100; + if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) + { + int64_t d = winfo->lastTs - rand() % 1000000 + rand_num; + //generateData(data, datatype, ncols_per_record, d, len_of_binary); + (void)generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo); + } else { + //generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); + (void)generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo); + } + pstr += sprintf(pstr, "%s", data); + winfo->counter++; + + if (winfo->counter >= winfo->superTblInfo->insertRows) { + break; + } + } + + if (g_args.insert_interval) { + winfo->st = taosGetTimestampMs(); + } + taos_query_a(winfo->taos, buffer, callBack, winfo); + free(buffer); + free(data); + + taos_free_result(res); +} + +void *asyncWrite(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + winfo->st = 0; + winfo->et = 0; + winfo->lastTs = winfo->start_time; + + if (g_args.insert_interval) { + winfo->st = taosGetTimestampMs(); + } + taos_query_a(winfo->taos, "show databases", callBack, winfo); + + tsem_wait(&(winfo->lock_sem)); + + return NULL; +} + +void startMultiThreadInsertData(int threads, char* db_name, char* precision, + SSuperTable* superTblInfo) { + + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); + + int ntables = 0; + if (superTblInfo) + ntables = superTblInfo->childTblCount; + else + ntables = g_args.num_of_tables; - if (!winfo->use_metric) { - /* Create all the tables; */ - printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); - for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols); - queryDB(winfo->taos, command); + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; } - } else { - /* Create all the tables; */ - printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); - for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - int j; - if (i % 10 == 0) { - j = 10; - } else { - j = i % 10; - } - if (j % 2 == 0) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.meters tags (%d,\"%s\");", winfo->db_name, winfo->tb_prefix, i, winfo->db_name, j, "shanghai"); - } else { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.meters tags (%d,\"%s\");", winfo->db_name, winfo->tb_prefix, i, winfo->db_name, j, "beijing"); + + int b = 0; + if (threads != 0) { + b = ntables % threads; } - queryDB(winfo->taos, command); + + //TAOS* taos; + //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + // taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + // if (NULL == taos) { + // printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); + // exit(-1); + // } + //} + + int32_t timePrec = TSDB_TIME_PRECISION_MILLI; + if (0 != precision[0]) { + if (0 == strncasecmp(precision, "ms", 2)) { + timePrec = TSDB_TIME_PRECISION_MILLI; + } else if (0 == strncasecmp(precision, "us", 2)) { + timePrec = TSDB_TIME_PRECISION_MICRO; + } else { + printf("No support precision: %s\n", precision); + exit(-1); } } - return NULL; -} - -void multiThreadCreateTable(char* cols, bool use_metric, int threads, int ntables, char* db_name, char* tb_prefix, char *ip_addr, uint16_t port, char *user, char *pass) { - double ts = getCurrentTime(); - printf("create table......\n"); - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - info *infos = malloc(threads * sizeof(info)); - - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; + int64_t start_time; + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + start_time = taosGetTimestamp(timePrec); + } else { + if (TSDB_CODE_SUCCESS != taosParseTime( + superTblInfo->startTimestamp, + &start_time, + strlen(superTblInfo->startTimestamp), + timePrec, 0)) { + printf("ERROR to parse time!\n"); + exit(-1); + } + } + } else { + start_time = 1500000000000; } - int b = 0; - if (threads != 0) - b = ntables % threads; + double start = getCurrentTime(); + int last = 0; for (int i = 0; i < threads; i++) { - info *t_info = infos + i; + threadInfo *t_info = infos + i; t_info->threadID = i; tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - tstrncpy(t_info->tb_prefix, tb_prefix, MAX_TB_NAME_SIZE); - t_info->taos = taos_connect(ip_addr, user, pass, db_name, port); - t_info->start_table_id = last; - t_info->end_table_id = i < b ? last + a : last + a - 1; - last = t_info->end_table_id + 1; - t_info->use_metric = use_metric; - t_info->cols = cols; - pthread_create(pids + i, NULL, createTable, t_info); + t_info->superTblInfo = superTblInfo; + + t_info->start_time = start_time; + t_info->minDelay = INT16_MAX; + + if ((NULL == superTblInfo) || + (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) { + //t_info->taos = taos; + t_info->taos = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); + if (NULL == t_info->taos) { + printf("connect to server fail from insert sub thread, reason: %s\n", + taos_errstr(NULL)); + exit(-1); + } + } else { + t_info->taos = NULL; + } + + if ((NULL == superTblInfo) + || (0 == superTblInfo->multiThreadWriteOneTbl)) { + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + last = t_info->end_table_id + 1; + } else { + t_info->start_table_id = 0; + t_info->end_table_id = superTblInfo->childTblCount - 1; + t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); + } + + tsem_init(&(t_info->lock_sem), 0, 0); + if (SYNC == g_Dbs.queryMode) { + if (superTblInfo) { + pthread_create(pids + i, NULL, syncWriteWithStb, t_info); + } else { + pthread_create(pids + i, NULL, syncWrite, t_info); + } + } else { + pthread_create(pids + i, NULL, asyncWrite, t_info); + } } for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } - double t = getCurrentTime() - ts; - printf("Spent %.4f seconds to create %d tables with %d connections\n", t, ntables, threads); + int64_t totalDelay = 0; + int64_t maxDelay = 0; + int64_t minDelay = INT16_MAX; + int64_t cntDelay = 1; + double avgDelay = 0; for (int i = 0; i < threads; i++) { - info *t_info = infos + i; - tsem_destroy(&(t_info->mutex_sem)); + threadInfo *t_info = infos + i; + tsem_destroy(&(t_info->lock_sem)); + taos_close(t_info->taos); + + if (superTblInfo) { + superTblInfo->totalAffectedRows += t_info->totalAffectedRows; + superTblInfo->totalRowsInserted += t_info->totalRowsInserted; + + totalDelay += t_info->totalDelay; + cntDelay += t_info->cntDelay; + if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; + if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; + } } + cntDelay -= 1; + + if (cntDelay == 0) cntDelay = 1; + avgDelay = (double)totalDelay / cntDelay; + + double end = getCurrentTime(); + double t = end - start; + + if (superTblInfo) { + printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + t, superTblInfo->totalRowsInserted, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + superTblInfo->totalRowsInserted / t); + fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + t, superTblInfo->totalRowsInserted, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + superTblInfo->totalRowsInserted / t); + } + + printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + + //taos_close(taos); free(pids); free(infos); - - return ; } + void *readTable(void *sarg) { - info *rinfo = (info *)sarg; +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; - int64_t sTime = rinfo->start_time; + uint64_t sTime = rinfo->start_time; char *tb_prefix = rinfo->tb_prefix; FILE *fp = fopen(rinfo->fp, "a"); if (NULL == fp) { printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); return NULL; } - - int num_of_DPT = rinfo->nrecords_per_table; + + int num_of_DPT; + if (rinfo->superTblInfo) { + num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; + } else { + num_of_DPT = g_args.num_of_DPT; + } + int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; int totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = rinfo->do_aggreFunc; + bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { @@ -1110,7 +4727,8 @@ void *readTable(void *sarg) { fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); taos_free_result(pSql); taos_close(taos); - exit(EXIT_FAILURE); + fclose(fp); + return NULL; } while (taos_fetch_row(pSql) != NULL) { @@ -1129,13 +4747,14 @@ void *readTable(void *sarg) { printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT); } fprintf(fp, "\n"); - fclose(fp); +#endif return NULL; } void *readMetric(void *sarg) { - info *rinfo = (info *)sarg; +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; FILE *fp = fopen(rinfo->fp, "a"); @@ -1143,11 +4762,11 @@ void *readMetric(void *sarg) { printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); return NULL; } - - int num_of_DPT = rinfo->nrecords_per_table; + + int num_of_DPT = rinfo->superTblInfo->insertRows; int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; int totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = rinfo->do_aggreFunc; + bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { @@ -1164,9 +4783,9 @@ void *readMetric(void *sarg) { for (int i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "areaid = %d", i); + sprintf(tempS, "t1 = %d", i); } else { - sprintf(tempS, " or areaid = %d ", i); + sprintf(tempS, " or t1 = %d ", i); } strcat(condition, tempS); @@ -1184,7 +4803,8 @@ void *readMetric(void *sarg) { fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); taos_free_result(pSql); taos_close(taos); - exit(1); + fclose(fp); + return NULL; } int count = 0; while (taos_fetch_row(pSql) != NULL) { @@ -1199,284 +4819,918 @@ void *readMetric(void *sarg) { } fprintf(fp, "\n"); } - fclose(fp); +#endif return NULL; } -void queryDB(TAOS *taos, char *command) { - int i; - TAOS_RES *pSql = NULL; - int32_t code = -1; - for (i = 0; i < 5; i++) { - if (NULL != pSql) { - taos_free_result(pSql); - pSql = NULL; +int insertTestProcess() { + + setupForAnsiEscape(); + int ret = printfInsertMeta(); + resetAfterAnsiEscape(); + + if (ret == -1) + exit(EXIT_FAILURE); + + debugPrint("DEBUG - %d result file: %s\n", __LINE__, g_Dbs.resultFile); + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); + if (NULL == g_fpOfInsertResult) { + fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); + return -1; + } { + printfInsertMetaToFile(g_fpOfInsertResult); + } + + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } + + init_rand_data(); + + // create database and super tables + if(createDatabases() != 0) { + fclose(g_fpOfInsertResult); + return -1; + } + + // pretreatement + prePareSampleData(); + + double start; + double end; + + // create child tables + start = getCurrentTime(); + createChildTables(); + end = getCurrentTime(); + + if (g_totalChildTables > 0) { + printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + end - start, g_totalChildTables, g_Dbs.threadCount); + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + end - start, g_totalChildTables, g_Dbs.threadCount); + } + + taosMsleep(1000); + // create sub threads for inserting data + //start = getCurrentTime(); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].superTblCount > 0) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + if (0 == g_Dbs.db[i].superTbls[j].insertRows) { + continue; + } + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); + } + } else { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + NULL); + } + } + //end = getCurrentTime(); + + //int64_t totalRowsInserted = 0; + //int64_t totalAffectedRows = 0; + //for (int i = 0; i < g_Dbs.dbCount; i++) { + // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // totalRowsInserted += g_Dbs.db[i].superTbls[j].totalRowsInserted; + // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; + //} + //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalRowsInserted, totalAffectedRows, g_Dbs.threadCount); + postFreeResource(); + + return 0; +} + +void *superQueryProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + //char sqlStr[MAX_TB_NAME_SIZE*2]; + //sprintf(sqlStr, "use %s", g_queryInfo.dbName); + //queryDB(winfo->taos, sqlStr); + + int64_t st = 0; + int64_t et = 0; + while (1) { + if (g_queryInfo.superQueryInfo.rate && (et - st) < (int64_t)g_queryInfo.superQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + st = taosGetTimestampMs(); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + int64_t t1 = taosGetTimestampUs(); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); + } + selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], tmpFile); + int64_t t2 = taosGetTimestampUs(); + printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + } else { + int64_t t1 = taosGetTimestampUs(); + int retCode = postProceSql(g_queryInfo.host, + g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i]); + int64_t t2 = taosGetTimestampUs(); + printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + + if (0 != retCode) { + printf("====restful return fail, threadID[%d]\n", winfo->threadID); + return NULL; + } + } + } + et = taosGetTimestampMs(); + printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", + taosGetSelfPthreadId(), (double)(et - st)/1000.0); + } + return NULL; +} + +void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { + char sourceString[32] = "xxxx"; + char subTblName[MAX_TB_NAME_SIZE*3]; + sprintf(subTblName, "%s.%s", + g_queryInfo.dbName, + g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); + + //printf("inSql: %s\n", inSql); + + char* pos = strstr(inSql, sourceString); + if (0 == pos) { + return; + } + + tstrncpy(outSql, inSql, pos - inSql + 1); + //printf("1: %s\n", outSql); + strcat(outSql, subTblName); + //printf("2: %s\n", outSql); + strcat(outSql, pos+strlen(sourceString)); + //printf("3: %s\n", outSql); +} + +void *subQueryProcess(void *sarg) { + char sqlstr[1024]; + threadInfo *winfo = (threadInfo *)sarg; + int64_t st = 0; + int64_t et = (int64_t)g_queryInfo.subQueryInfo.rate*1000; + while (1) { + if (g_queryInfo.subQueryInfo.rate && (et - st) < g_queryInfo.subQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.subQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + st = taosGetTimestampMs(); + for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { + for (int j = 0; j < g_queryInfo.subQueryInfo.sqlCount; j++) { + memset(sqlstr,0,sizeof(sqlstr)); + replaceSubTblName(g_queryInfo.subQueryInfo.sql[j], sqlstr, i); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.subQueryInfo.result[i], + winfo->threadID); + } + selectAndGetResult(winfo->taos, sqlstr, tmpFile); + } + } + et = taosGetTimestampMs(); + printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", + taosGetSelfPthreadId(), + winfo->start_table_id, + winfo->end_table_id, + (double)(et - st)/1000.0); + } + return NULL; +} + +static int queryTestProcess() { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.subQueryInfo.sqlCount) { + (void)getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.subQueryInfo.sTblName, + &g_queryInfo.subQueryInfo.childTblName, + &g_queryInfo.subQueryInfo.childTblCount); + } + + printfQueryMeta(); + + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } + + printfQuerySystemInfo(taos); + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from specify table + if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { + + pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); } - pSql = taos_query(taos, command); - code = taos_errno(pSql); - if (0 == code) { - break; - } + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + t_info->taos = taos; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + debugPrint("DEBUG %s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + (void)queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE); + } else { + t_info->taos = NULL; + } + + pthread_create(pids + i, NULL, superQueryProcess, t_info); + } + }else { + g_queryInfo.superQueryInfo.concurrent = 0; + } + + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + //==== create sub threads for query from all sub table of the super table + if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + int ntables = g_queryInfo.subQueryInfo.childTblCount; + int threads = g_queryInfo.subQueryInfo.threadCnt; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + last = t_info->end_table_id + 1; + t_info->taos = taos; + pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); + } + + g_queryInfo.subQueryInfo.threadCnt = threads; + }else { + g_queryInfo.subQueryInfo.threadCnt = 0; + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); } - if (code != 0) { - fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); - taos_free_result(pSql); - taos_close(taos); - exit(EXIT_FAILURE); + tmfree((char*)pids); + tmfree((char*)infos); + + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); + + taos_close(taos); + return 0; +} + +static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + printf("failed to subscribe result, code:%d, reason:%s\n", code, taos_errstr(res)); + return; + } + + getResult(res, (char*)param); + taos_free_result(res); +} + +static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { + TAOS_SUB* tsub = NULL; + + if (g_queryInfo.superQueryInfo.subscribeMode) { + tsub = taos_subscribe(taos, + g_queryInfo.superQueryInfo.subscribeRestart, + topic, sql, subscribe_callback, (void*)resultFileName, + g_queryInfo.superQueryInfo.subscribeInterval); + } else { + tsub = taos_subscribe(taos, + g_queryInfo.superQueryInfo.subscribeRestart, + topic, sql, NULL, NULL, 0); + } + + if (tsub == NULL) { + printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql); + return NULL; + } + + return tsub; +} + +void *subSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + char subSqlstr[1024]; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + debugPrint("DEBUG %s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){ + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + memset(subSqlstr,0,sizeof(subSqlstr)); + replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); + } + g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); + if (NULL == g_queryInfo.subQueryInfo.tsub[i]) { + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + TAOS_RES* res = NULL; + while (1) { + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.subQueryInfo.subscribeMode) { + continue; + } + + res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]); + if (res) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.subQueryInfo.result[i], + winfo->threadID); + } + getResult(res, tmpFile); + } + } + } + taos_free_result(res); + + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], + g_queryInfo.subQueryInfo.subscribeKeepProgress); + } + return NULL; +} + +void *superSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + debugPrint("DEBUG %s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) { + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.subQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[i], winfo->threadID); + } + g_queryInfo.superQueryInfo.tsub[i] = + subscribeImpl(winfo->taos, + g_queryInfo.superQueryInfo.sql[i], + topic, tmpFile); + if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + TAOS_RES* res = NULL; + while (1) { + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.superQueryInfo.subscribeMode) { + continue; + } + + res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); + if (res) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[i], winfo->threadID); + } + getResult(res, tmpFile); + } + } + } + taos_free_result(res); + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + return NULL; +} + +static int subscribeTestProcess() { + printfQueryMeta(); + + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } + + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.subQueryInfo.sqlCount) { + (void)getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.subQueryInfo.sTblName, + &g_queryInfo.subQueryInfo.childTblName, + &g_queryInfo.subQueryInfo.childTblCount); + } + + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from super table + if (g_queryInfo.superQueryInfo.sqlCount > 0 + && g_queryInfo.superQueryInfo.concurrent > 0) { + pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + t_info->taos = taos; + pthread_create(pids + i, NULL, superSubscribeProcess, t_info); + } + } + + //==== create sub threads for query from sub table + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + if ((g_queryInfo.subQueryInfo.sqlCount > 0) + && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * + sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * + sizeof(threadInfo)); + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + int ntables = g_queryInfo.subQueryInfo.childTblCount; + int threads = g_queryInfo.subQueryInfo.threadCnt; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + t_info->taos = taos; + pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); + } + g_queryInfo.subQueryInfo.threadCnt = threads; + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); + } + + tmfree((char*)pids); + tmfree((char*)infos); + + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); + taos_close(taos); + return 0; +} + +void initOfInsertMeta() { + memset(&g_Dbs, 0, sizeof(SDbs)); + + // set default values + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_Dbs.port = 6030; + tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); + g_Dbs.threadCount = 2; + g_Dbs.use_metric = true; +} + +void initOfQueryMeta() { + memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); + + // set default values + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_queryInfo.port = 6030; + tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); +} + +void setParaFromArg(){ + if (g_args.host) { + strcpy(g_Dbs.host, g_args.host); + } else { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); } - taos_free_result(pSql); -} + if (g_args.user) { + strcpy(g_Dbs.user, g_args.user); + } -// sync insertion -void *syncWrite(void *sarg) { - info *winfo = (info *)sarg; - char buffer[BUFFER_SIZE] = "\0"; - char data[MAX_DATA_SIZE]; - char **data_type = winfo->datatype; - int len_of_binary = winfo->len_of_binary; - int ncols_per_record = winfo->ncols_per_record; - srand((uint32_t)time(NULL)); - int64_t time_counter = winfo->start_time; - for (int i = 0; i < winfo->nrecords_per_table;) { - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { - int inserted = i; - int64_t tmp_time = time_counter; + if (g_args.password) { + strcpy(g_Dbs.password, g_args.password); + } + + if (g_args.port) { + g_Dbs.port = g_args.port; + } - char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, tID); - int k; - for (k = 0; k < winfo->nrecords_per_request;) { - int rand_num = rand() % 100; - int len = -1; - if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) { - int64_t d = tmp_time - rand() % 1000000 + rand_num; - len = generateData(data, data_type, ncols_per_record, d, len_of_binary); - } else { - len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary); - } + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - //assert(len + pstr - buffer < BUFFER_SIZE); - if (len + pstr - buffer >= BUFFER_SIZE) { // too long - break; - } + g_Dbs.dbCount = 1; + g_Dbs.db[0].drop = 1; - pstr += sprintf(pstr, " %s", data); - inserted++; - k++; + tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); + g_Dbs.db[0].dbCfg.replica = g_args.replica; + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - if (inserted >= winfo->nrecords_per_table) break; - } + tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); - /* puts(buffer); */ - queryDB(winfo->taos, buffer); + g_Dbs.use_metric = g_args.use_metric; + g_Dbs.insert_only = g_args.insert_only; - if (tID == winfo->end_table_id) { - i = inserted; - time_counter = tmp_time; + g_Dbs.do_aggreFunc = true; + + char dataString[STRING_LEN]; + char **data_type = g_args.datatype; + + memset(dataString, 0, STRING_LEN); + + if (strcasecmp(data_type[0], "BINARY") == 0 + || strcasecmp(data_type[0], "BOOL") == 0 + || strcasecmp(data_type[0], "NCHAR") == 0 ) { + g_Dbs.do_aggreFunc = false; + } + + if (g_args.use_metric) { + g_Dbs.db[0].superTblCount = 1; + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = 1; + g_Dbs.queryMode = g_args.mode; + + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; + g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; + tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, + g_args.tb_prefix, MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, + "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].timeStampStep = 10; + + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; + + g_Dbs.db[0].superTbls[0].columnCount = 0; + for (int i = 0; i < MAX_NUM_DATATYPE; i++) { + if (data_type[i] == NULL) { + break; + } + + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + data_type[i], MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columnCount++; + } + + if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { + g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; + } else { + for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount++; } } + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tagCount = 2; + } else { + g_Dbs.threadCountByCreateTbl = 1; + g_Dbs.db[0].superTbls[0].tagCount = 0; } - return NULL; + } -void *asyncWrite(void *sarg) { - info *winfo = (info *)sarg; - taos_query_a(winfo->taos, "show databases", callBack, winfo); +/* Function to do regular expression check */ +static int regexMatch(const char *s, const char *reg, int cflags) { + regex_t regex; + char msgbuf[100] = {0}; - tsem_wait(&(winfo->lock_sem)); + /* Compile regular expression */ + if (regcomp(®ex, reg, cflags) != 0) { + printf("Fail to compile regex\n"); + exit(-1); + } - return NULL; + /* Execute regular expression */ + int reti = regexec(®ex, s, 0, NULL, 0); + if (!reti) { + regfree(®ex); + return 1; + } else if (reti == REG_NOMATCH) { + regfree(®ex); + return 0; + } else { + regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); + printf("Regex match failed: %s\n", msgbuf); + regfree(®ex); + exit(-1); + } + + return 0; } -void callBack(void *param, TAOS_RES *res, int code) { - info* winfo = (info*)param; - char **datatype = winfo->datatype; - int ncols_per_record = winfo->ncols_per_record; - int len_of_binary = winfo->len_of_binary; +static int isCommentLine(char *line) { + if (line == NULL) return 1; - int64_t tmp_time = winfo->start_time; - char *buffer = calloc(1, BUFFER_SIZE); - char *data = calloc(1, MAX_DATA_SIZE); - char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id); - if (winfo->counter >= winfo->nrecords_per_table) { - winfo->start_table_id++; - winfo->counter = 0; - } - if (winfo->start_table_id > winfo->end_table_id) { - tsem_post(&winfo->lock_sem); - free(buffer); - free(data); - taos_free_result(res); + return regexMatch(line, "^\\s*#.*", REG_EXTENDED); +} + +void querySqlFile(TAOS* taos, char* sqlFile) +{ + FILE *fp = fopen(sqlFile, "r"); + if (fp == NULL) { + printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); return; } - - for (int i = 0; i < winfo->nrecords_per_request; i++) { - int rand_num = rand() % 100; - if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) - { - int64_t d = tmp_time - rand() % 1000000 + rand_num; - generateData(data, datatype, ncols_per_record, d, len_of_binary); - } else - { - generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); - } - pstr += sprintf(pstr, "%s", data); - winfo->counter++; - if (winfo->counter >= winfo->nrecords_per_table) { - break; + int read_len = 0; + char * cmd = calloc(1, MAX_SQL_SIZE); + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; + + double t = getCurrentTime(); + + while ((read_len = tgetline(&line, &line_len, fp)) != -1) { + if (read_len >= MAX_SQL_SIZE) continue; + line[--read_len] = '\0'; + + if (read_len == 0 || isCommentLine(line)) { // line starts with # + continue; } - } - taos_query_a(winfo->taos, buffer, callBack, winfo); - free(buffer); - free(data); - taos_free_result(res); -} + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } -double getCurrentTime() { - struct timeval tv; - if (gettimeofday(&tv, NULL) != 0) { - perror("Failed to get current time in ms"); - exit(EXIT_FAILURE); + memcpy(cmd + cmd_len, line, read_len); + debugPrint("DEBUG %s() LN%d cmd: %s\n", __func__, __LINE__, cmd); + queryDbExec(taos, cmd, NO_INSERT_TYPE); + memset(cmd, 0, MAX_SQL_SIZE); + cmd_len = 0; } - return tv.tv_sec + tv.tv_usec / 1E6; + t = getCurrentTime() - t; + printf("run %s took %.6f second(s)\n\n", sqlFile, t); + + tmfree(cmd); + tmfree(line); + tmfclose(fp); + return; } -int32_t randint[MAX_PREPARED_RAND]; -int64_t randbigint[MAX_PREPARED_RAND]; -float randfloat[MAX_PREPARED_RAND]; -double randdouble[MAX_PREPARED_RAND]; +static void testMetaFile() { + if (INSERT_MODE == g_args.test_mode) { + if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); + insertTestProcess(); + } else if (QUERY_MODE == g_args.test_mode) { + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + queryTestProcess(); + } else if (SUBSCRIBE_MODE == g_args.test_mode) { + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + subscribeTestProcess(); + } else { + ; + } +} -int32_t rand_tinyint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 128; +static void testCmdLine() { -} + g_args.test_mode = INSERT_MODE; + insertTestProcess(); -int32_t rand_smallint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 32767; -} + if (g_Dbs.insert_only) + return; -int32_t rand_int(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor]; -} + // select + if (false == g_Dbs.insert_only) { + // query data -int64_t rand_bigint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randbigint[cursor]; - -} + pthread_t read_id; + threadInfo *rInfo = malloc(sizeof(threadInfo)); + rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + rInfo->start_table_id = 0; -float rand_float(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randfloat[cursor]; - -} + //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + if (g_args.use_metric) { + rInfo->end_table_id = g_Dbs.db[0].superTbls[0].childTblCount - 1; + rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + strcpy(rInfo->tb_prefix, + g_Dbs.db[0].superTbls[0].childTblPrefix); + } else { + rInfo->end_table_id = g_args.num_of_tables -1; + strcpy(rInfo->tb_prefix, g_args.tb_prefix); + } -double rand_double() { - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randdouble[cursor]; + rInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + if (rInfo->taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + free(rInfo); + exit(-1); + } -} + strcpy(rInfo->fp, g_Dbs.resultFile); -void init_rand_data(){ - for (int i = 0; i < MAX_PREPARED_RAND; i++){ - randint[i] = (int)(rand() % 10); - randbigint[i] = (int64_t)(rand() % 2147483648); - randfloat[i] = (float)(rand() / 1000.0); - randdouble[i] = (double)(rand() / 1000000.0); - } + if (!g_Dbs.use_metric) { + pthread_create(&read_id, NULL, readTable, rInfo); + } else { + pthread_create(&read_id, NULL, readMetric, rInfo); + } + pthread_join(read_id, NULL); + taos_close(rInfo->taos); + free(rInfo); + } } -int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) { - memset(res, 0, MAX_DATA_SIZE); - char *pstr = res; - pstr += sprintf(pstr, "(%" PRId64, timestamp); - int c = 0; +int main(int argc, char *argv[]) { + parse_args(argc, argv, &g_args); - for (; c < MAX_NUM_DATATYPE; c++) { - if (data_type[c] == NULL) { - break; - } - } + debugPrint("DEBUG - meta file: %s\n", g_args.metaFile); - if (0 == c) { - perror("data type error!"); - exit(-1); - } + if (g_args.metaFile) { + initOfInsertMeta(); + initOfQueryMeta(); - for (int i = 0; i < num_of_cols; i++) { - if (strcasecmp(data_type[i % c], "tinyint") == 0) { - pstr += sprintf(pstr, ", %d", rand_tinyint() ); - } else if (strcasecmp(data_type[i % c], "smallint") == 0) { - pstr += sprintf(pstr, ", %d", rand_smallint()); - } else if (strcasecmp(data_type[i % c], "int") == 0) { - pstr += sprintf(pstr, ", %d", rand_int()); - } else if (strcasecmp(data_type[i % c], "bigint") == 0) { - pstr += sprintf(pstr, ", %" PRId64, rand_bigint()); - } else if (strcasecmp(data_type[i % c], "float") == 0) { - pstr += sprintf(pstr, ", %10.4f", rand_float()); - } else if (strcasecmp(data_type[i % c], "double") == 0) { - double t = rand_double(); - pstr += sprintf(pstr, ", %20.8f", t); - } else if (strcasecmp(data_type[i % c], "bool") == 0) { - bool b = rand() & 1; - pstr += sprintf(pstr, ", %s", b ? "true" : "false"); - } else if (strcasecmp(data_type[i % c], "binary") == 0) { - char *s = malloc(len_of_binary); - rand_string(s, len_of_binary); - pstr += sprintf(pstr, ", \"%s\"", s); - free(s); - }else if (strcasecmp(data_type[i % c], "nchar") == 0) { - char *s = malloc(len_of_binary); - rand_string(s, len_of_binary); - pstr += sprintf(pstr, ", \"%s\"", s); - free(s); + if (false == getInfoFromJsonFile(g_args.metaFile)) { + printf("Failed to read %s\n", g_args.metaFile); + return 1; } - if (pstr - res > MAX_DATA_SIZE) { - perror("column length too long, abort"); - exit(-1); + testMetaFile(); + } else { + memset(&g_Dbs, 0, sizeof(SDbs)); + setParaFromArg(); + + if (NULL != g_args.sqlFile) { + TAOS* qtaos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + querySqlFile(qtaos, g_args.sqlFile); + taos_close(qtaos); + + } else { + testCmdLine(); } } - pstr += sprintf(pstr, ")"); - - return (int32_t)(pstr - res); + return 0; } -static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890"; -void rand_string(char *str, int size) { - str[0] = 0; - if (size > 0) { - --size; - int n; - for (n = 0; n < size; n++) { - int key = rand() % (int)(sizeof charset - 1); - str[n] = charset[key]; - } - str[n] = 0; - } -} diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index dcdbd486154b2673459e25754dc75457851a4df4..58897b89e95743c802755c0476f3b2843a244a59 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -12,5 +12,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(taosdump taos_static) ELSE () TARGET_LINK_LIBRARIES(taosdump taos) - ENDIF () + ENDIF () +ENDIF () + +IF (TD_DARWIN) + # missing for macosx + # ADD_EXECUTABLE(taosdump ${SRC}) + # IF (TD_SOMODE_STATIC) + # TARGET_LINK_LIBRARIES(taosdump taos_static) + # ELSE () + # TARGET_LINK_LIBRARIES(taosdump taos) + # ENDIF () ENDIF () diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index a7258c97241f1b0f629f168952a9d98d60d343ff..3cee5f1b1dc02e0218c97e66c4af9acb6c77c69f 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -14,6 +14,9 @@ */ #include +#include +#include + #include "os.h" #include "taos.h" #include "taosdef.h" @@ -64,7 +67,10 @@ enum _show_tables_index { TSDB_SHOW_TABLES_NAME_INDEX, TSDB_SHOW_TABLES_CREATED_TIME_INDEX, TSDB_SHOW_TABLES_COLUMNS_INDEX, - TSDB_SHOW_TABLES_METRIC_INDEX, + TSDB_SHOW_TABLES_METRIC_INDEX, + TSDB_SHOW_TABLES_UID_INDEX, + TSDB_SHOW_TABLES_TID_INDEX, + TSDB_SHOW_TABLES_VGID_INDEX, TSDB_MAX_SHOW_TABLES }; @@ -92,24 +98,27 @@ typedef struct { extern char version[]; typedef struct { - char name[TSDB_DB_NAME_LEN + 1]; - int32_t tables; + char name[TSDB_DB_NAME_LEN + 1]; + char create_time[32]; + int32_t ntables; int32_t vgroups; - int16_t replications; + int16_t replica; int16_t quorum; - int16_t daysPerFile; - int16_t daysToKeep; - int16_t daysToKeep1; - int16_t daysToKeep2; - int32_t cacheBlockSize; //MB - int32_t totalBlocks; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; - int8_t walLevel; - int32_t fsyncPeriod; - int8_t compression; - int8_t precision; // time resolution + int16_t days; + char keeplist[32]; + //int16_t daysToKeep; + //int16_t daysToKeep1; + //int16_t daysToKeep2; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + char precision[8]; // time resolution int8_t update; + char status[16]; } SDbInfo; typedef struct { @@ -128,8 +137,17 @@ typedef struct { int32_t totalThreads; char dbName[TSDB_TABLE_NAME_LEN + 1]; void *taosCon; + int64_t rowsOfDumpOut; + int64_t tablesOfDumpOut; } SThreadParaObj; +typedef struct { + int64_t totalRowsOfDumpOut; + int64_t totalChildTblsOfDumpOut; + int32_t totalSuperTblsOfDumpOut; + int32_t totalDatabasesOfDumpOut; +} resultStatistics; + static int64_t totalDumpOutRows = 0; SDbInfo **dbInfos = NULL; @@ -167,6 +185,7 @@ static struct argp_option options[] = { // input/output file {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, + {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, #ifdef _TD_POWER_ {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, #else @@ -200,6 +219,8 @@ struct arguments { // output file char outpath[TSDB_FILENAME_LEN+1]; char inpath[TSDB_FILENAME_LEN+1]; + // result file + char *resultFile; char *encode; // dump unit option bool all_databases; @@ -274,6 +295,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { tstrncpy(arguments->inpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; + case 'r': + arguments->resultFile = arg; + break; case 'c': if (wordexp(arg, &full_path, 0) != 0) { fprintf(stderr, "Invalid path %s\n", arg); @@ -308,6 +332,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'N': arguments->data_batch = atoi(arg); + if (arguments->data_batch >= INT16_MAX) { + arguments->data_batch = INT16_MAX - 1; + } break; case 'L': { @@ -343,19 +370,22 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Our argp parser. */ static struct argp argp = {options, parse_opt, args_doc, doc}; +static resultStatistics g_resultStatistics = {0}; +static FILE *g_fpOfResult = NULL; +static int g_numOfCores = 1; int taosDumpOut(struct arguments *arguments); int taosDumpIn(struct arguments *arguments); void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); -int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon); -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp); -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp); -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon); -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon); +int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); +int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); +int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); int taosCheckParam(struct arguments *arguments); void taosFreeDbInfos(); -static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName); +static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); struct arguments tsArguments = { // connection option @@ -371,7 +401,8 @@ struct arguments tsArguments = { 0, // outpath and inpath "", - "", + "", + "./dump_result.txt", NULL, // dump unit option false, @@ -392,18 +423,34 @@ struct arguments tsArguments = { 0, false }; - -int queryDB(TAOS *taos, char *command) { - TAOS_RES *pSql = NULL; + +static int queryDbImpl(TAOS *taos, char *command) { + int i; + TAOS_RES *res = NULL; int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } - pSql = taos_query(taos, command); - code = taos_errno(pSql); - if (code) { - fprintf(stderr, "sql error: %s, reason:%s\n", command, taos_errstr(pSql)); - } - taos_free_result(pSql); - return code; + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run <%s>, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + taos_free_result(res); + return 0; } int main(int argc, char *argv[]) { @@ -430,6 +477,7 @@ int main(int argc, char *argv[]) { printf("mysqlFlag: %d\n", tsArguments.mysqlFlag); printf("outpath: %s\n", tsArguments.outpath); printf("inpath: %s\n", tsArguments.inpath); + printf("resultFile: %s\n", tsArguments.resultFile); printf("encode: %s\n", tsArguments.encode); printf("all_databases: %d\n", tsArguments.all_databases); printf("databases: %d\n", tsArguments.databases); @@ -459,13 +507,80 @@ int main(int argc, char *argv[]) { if (taosCheckParam(&tsArguments) < 0) { exit(EXIT_FAILURE); } + + g_fpOfResult = fopen(tsArguments.resultFile, "a"); + if (NULL == g_fpOfResult) { + fprintf(stderr, "Failed to open %s for save result\n", tsArguments.resultFile); + return 1; + }; + + fprintf(g_fpOfResult, "#############################################################################\n"); + fprintf(g_fpOfResult, "============================== arguments config =============================\n"); + { + fprintf(g_fpOfResult, "host: %s\n", tsArguments.host); + fprintf(g_fpOfResult, "user: %s\n", tsArguments.user); + fprintf(g_fpOfResult, "password: %s\n", tsArguments.password); + fprintf(g_fpOfResult, "port: %u\n", tsArguments.port); + fprintf(g_fpOfResult, "cversion: %s\n", tsArguments.cversion); + fprintf(g_fpOfResult, "mysqlFlag: %d\n", tsArguments.mysqlFlag); + fprintf(g_fpOfResult, "outpath: %s\n", tsArguments.outpath); + fprintf(g_fpOfResult, "inpath: %s\n", tsArguments.inpath); + fprintf(g_fpOfResult, "resultFile: %s\n", tsArguments.resultFile); + fprintf(g_fpOfResult, "encode: %s\n", tsArguments.encode); + fprintf(g_fpOfResult, "all_databases: %d\n", tsArguments.all_databases); + fprintf(g_fpOfResult, "databases: %d\n", tsArguments.databases); + fprintf(g_fpOfResult, "schemaonly: %d\n", tsArguments.schemaonly); + fprintf(g_fpOfResult, "with_property: %d\n", tsArguments.with_property); + fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", tsArguments.start_time); + fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", tsArguments.end_time); + fprintf(g_fpOfResult, "data_batch: %d\n", tsArguments.data_batch); + fprintf(g_fpOfResult, "max_sql_len: %d\n", tsArguments.max_sql_len); + fprintf(g_fpOfResult, "table_batch: %d\n", tsArguments.table_batch); + fprintf(g_fpOfResult, "thread_num: %d\n", tsArguments.thread_num); + fprintf(g_fpOfResult, "allow_sys: %d\n", tsArguments.allow_sys); + fprintf(g_fpOfResult, "abort: %d\n", tsArguments.abort); + fprintf(g_fpOfResult, "isDumpIn: %d\n", tsArguments.isDumpIn); + fprintf(g_fpOfResult, "arg_list_len: %d\n", tsArguments.arg_list_len); + + for (int32_t i = 0; i < tsArguments.arg_list_len; i++) { + fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, tsArguments.arg_list[i]); + } + } + + g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - if (tsArguments.isDumpIn) { - if (taosDumpIn(&tsArguments) < 0) return -1; + time_t tTime = time(NULL); + struct tm tm = *localtime(&tTime); + + if (tsArguments.isDumpIn) { + fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); + fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpIn(&tsArguments) < 0) { + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + return -1; + } } else { - if (taosDumpOut(&tsArguments) < 0) return -1; + fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); + fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpOut(&tsArguments) < 0) { + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + return -1; + } + + fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); + fprintf(g_fpOfResult, "# total database count: %d\n", g_resultStatistics.totalDatabasesOfDumpOut); + fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut); + fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut); } + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + return 0; } @@ -586,65 +701,100 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu sprintf(tmpCommand, "select tbname from %s", metric); - TAOS_RES *result = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(result); + TAOS_RES *res = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(res); if (code != 0) { fprintf(stderr, "failed to run command %s\n", tmpCommand); free(tmpCommand); - taos_free_result(result); + taos_free_result(res); return -1; } + free(tmpCommand); - TAOS_FIELD *fields = taos_fetch_fields(result); + char tmpBuf[TSDB_FILENAME_LEN + 1]; + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".select-tbname.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + taos_free_result(res); + return -1; + } - int32_t numOfTable = 0; - int32_t numOfThread = *totalNumOfThread; - char tmpFileName[TSDB_FILENAME_LEN + 1]; - while ((row = taos_fetch_row(result)) != NULL) { - if (0 == numOfTable) { - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, ".tables.tmp.%d", numOfThread); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(result); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - (void)remove(tmpFileName); - } - free(tmpCommand); - return -1; - } - - numOfThread++; - } + TAOS_FIELD *fields = taos_fetch_fields(res); + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { + memset(&tableRecord, 0, sizeof(STableRecord)); tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - + + taosWrite(fd, &tableRecord, sizeof(STableRecord)); numOfTable++; + } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); + + int maxThreads = arguments->thread_num; + int tableOfPerFile ; + if (numOfTable <= arguments->thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / arguments->thread_num; + if (0 != numOfTable % arguments->thread_num) { + tableOfPerFile += 1; + } + } - if (numOfTable >= arguments->table_batch) { - numOfTable = 0; + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; + } + + int32_t numOfThread = *totalNumOfThread; + int subFd = -1; + for (; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); + free(tblBuf); close(fd); - fd = -1; + return -1; } + + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; + } + taosWrite(subFd, tblBuf, readLen); + close(subFd); } + + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); if (fd >= 0) { close(fd); fd = -1; - } - - taos_free_result(result); + } *totalNumOfThread = numOfThread; - - free(tmpCommand); + free(tblBuf); return 0; } @@ -700,7 +850,7 @@ int taosDumpOut(struct arguments *arguments) { int32_t code = taos_errno(result); if (code != 0) { - fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos)); + fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(result)); goto _exit_failure; } @@ -736,27 +886,29 @@ int taosDumpOut(struct arguments *arguments) { } strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); -#if 0 if (arguments->with_property) { - dbInfos[count]->tables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); - dbInfos[count]->replications = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); + dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); - dbInfos[count]->daysPerFile = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); - dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); - dbInfos[count]->daysToKeep1; - dbInfos[count]->daysToKeep2; - dbInfos[count]->cacheBlockSize = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); - dbInfos[count]->totalBlocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); - dbInfos[count]->minRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); - dbInfos[count]->maxRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); - dbInfos[count]->walLevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); - dbInfos[count]->fsyncPeriod = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); - dbInfos[count]->compression = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); - dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); + dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + + strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); + //dbInfos[count]->daysToKeep1; + //dbInfos[count]->daysToKeep2; + dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); + dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); + dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); + dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); + dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); + dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); + dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + + strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); } -#endif count++; if (arguments->databases) { @@ -781,6 +933,8 @@ int taosDumpOut(struct arguments *arguments) { taosDumpDb(dbInfos[0], arguments, fp, taos); } else { // case: taosdump tablex tabley ... taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp); + fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfos[0]->name); + g_resultStatistics.totalDatabasesOfDumpOut++; sprintf(command, "use %s", dbInfos[0]->name); @@ -796,6 +950,7 @@ int taosDumpOut(struct arguments *arguments) { int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0 int normalTblFd = -1; int32_t retCode; + int superTblCnt = 0 ; for (int i = 1; arguments->arg_list[i]; i++) { if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) { fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]); @@ -803,11 +958,17 @@ int taosDumpOut(struct arguments *arguments) { } if (tableRecordInfo.isMetric) { // dump all table of this metric - (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos); + int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos, dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } retCode = taosSaveTableOfMetricToTempFile(taos, tableRecordInfo.tableRecord.metric, arguments, &totalNumOfThread); } else { if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric - (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos); + int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos, dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } } retCode = taosSaveAllNormalTableToTempFile(taos, tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, &normalTblFd); } @@ -819,13 +980,17 @@ int taosDumpOut(struct arguments *arguments) { goto _clean_tmp_file; } } + + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; if (-1 != normalTblFd){ taosClose(normalTblFd); } // start multi threads to dumpout - taosStartDumpOutWorkThreads(arguments, totalNumOfThread, dbInfos[0]->name); + taosStartDumpOutWorkThreads(taos, arguments, totalNumOfThread, dbInfos[0]->name); char tmpFileName[TSDB_FILENAME_LEN + 1]; _clean_tmp_file: @@ -855,41 +1020,27 @@ _exit_failure: return -1; } -int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { +int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { TAOS_ROW row = NULL; - TAOS_RES *tmpResult = NULL; + TAOS_RES* res = NULL; int count = 0; - char* tempCommand = (char *)malloc(COMMAND_SIZE); - if (tempCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } - - char* tbuf = (char *)malloc(COMMAND_SIZE); - if (tbuf == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - free(tempCommand); - return -1; - } - - sprintf(tempCommand, "describe %s", table); + char sqlstr[COMMAND_SIZE]; + sprintf(sqlstr, "describe %s.%s;", dbName, table); - tmpResult = taos_query(taosCon, tempCommand); - int32_t code = taos_errno(tmpResult); + res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tempCommand); - free(tempCommand); - free(tbuf); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + TAOS_FIELD *fields = taos_fetch_fields(res); tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN); - while ((row = taos_fetch_row(tmpResult)) != NULL) { + while ((row = taos_fetch_row(res)) != NULL) { strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], @@ -901,12 +1052,10 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe count++; } - taos_free_result(tmpResult); - tmpResult = NULL; + taos_free_result(res); + res = NULL; if (isSuperTable) { - free(tempCommand); - free(tbuf); return count; } @@ -915,37 +1064,33 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; - sprintf(tempCommand, "select %s from %s", tableDes->cols[i].field, table); + sprintf(sqlstr, "select %s from %s.%s", tableDes->cols[i].field, dbName, table); - tmpResult = taos_query(taosCon, tempCommand); - code = taos_errno(tmpResult); + res = taos_query(taosCon, sqlstr); + code = taos_errno(res); if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tempCommand); - free(tempCommand); - free(tbuf); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } - fields = taos_fetch_fields(tmpResult); + fields = taos_fetch_fields(res); - row = taos_fetch_row(tmpResult); + row = taos_fetch_row(res); if (NULL == row) { - fprintf(stderr, " fetch failed to run command %s\n", tempCommand); - free(tempCommand); - free(tbuf); - taos_free_result(tmpResult); + fprintf(stderr, " fetch failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } if (row[0] == NULL) { sprintf(tableDes->cols[i].note, "%s", "NULL"); - taos_free_result(tmpResult); - tmpResult = NULL; + taos_free_result(res); + res = NULL; continue; } - int32_t* length = taos_fetch_lengths(tmpResult); + int32_t* length = taos_fetch_lengths(res); //int32_t* length = taos_fetch_lengths(tmpResult); switch (fields[0].type) { @@ -970,18 +1115,22 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe case TSDB_DATA_TYPE_DOUBLE: sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0])); break; - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_BINARY: { memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); tableDes->cols[i].note[0] = '\''; + char tbuf[COMMAND_SIZE]; converStringToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE); char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf); *(pstr++) = '\''; break; - case TSDB_DATA_TYPE_NCHAR: + } + case TSDB_DATA_TYPE_NCHAR: { memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); + char tbuf[COMMAND_SIZE]; convertNCharToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE); sprintf(tableDes->cols[i].note, "\'%s\'", tbuf); break; + } case TSDB_DATA_TYPE_TIMESTAMP: sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); #if 0 @@ -1001,17 +1150,14 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe break; } - taos_free_result(tmpResult); - tmpResult = NULL; + taos_free_result(res); + res = NULL; } - free(tempCommand); - free(tbuf); - return count; } -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon) { +int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName) { int count = 0; STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); @@ -1030,7 +1176,7 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); */ - count = taosGetTableDes(table, tableDes, taosCon, false); + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); if (count < 0) { free(tableDes); @@ -1038,10 +1184,10 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI } // create child-table using super-table - taosDumpCreateMTableClause(tableDes, metric, count, fp); + taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName); } else { // dump table definition - count = taosGetTableDes(table, tableDes, taosCon, false); + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); if (count < 0) { free(tableDes); @@ -1049,39 +1195,28 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI } // create normal-table or super-table - taosDumpCreateTableClause(tableDes, count, fp); + taosDumpCreateTableClause(tableDes, count, fp, dbName); } free(tableDes); - return taosDumpTableData(fp, table, arguments, taosCon); + return taosDumpTableData(fp, table, arguments, taosCon, dbName); } void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return; - } - - char *pstr = tmpCommand; - - pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s", dbInfo->name); + char *pstr = sqlstr; + pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); if (isDumpProperty) { - #if 0 pstr += sprintf(pstr, - "TABLES %d vgroups %d REPLICA %d quorum %d DAYS %d KEEP %d CACHE %d BLOCKS %d MINROWS %d MAXROWS %d WALLEVEL %d FYNC %d COMP %d PRECISION %s UPDATE %d", - dbInfo->tables, dbInfo->vgroups, dbInfo->replications, dbInfo->quorum, dbInfo->daysPerFile, dbInfo->daysToKeep, dbInfo->cacheBlockSize, - dbInfo->totalBlocks, dbInfo->minRowsPerFileBlock, dbInfo->maxRowsPerFileBlock, dbInfo->walLevel, dbInfo->fsyncPeriod, dbInfo->compression, - dbInfo->precision, dbInfo->update); - #endif + "TABLES %d VGROUPS %d REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d WALLEVEL %d FYNC %d COMP %d PRECISION '%s' UPDATE %d", + dbInfo->ntables, dbInfo->vgroups, dbInfo->replica, dbInfo->quorum, dbInfo->days, dbInfo->keeplist, dbInfo->cache, + dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, dbInfo->wallevel, dbInfo->fsync, dbInfo->comp, dbInfo->precision, dbInfo->update); } pstr += sprintf(pstr, ";"); - - fprintf(fp, "%s\n\n", tmpCommand); - free(tmpCommand); + fprintf(fp, "%s\n\n", sqlstr); } void* taosDumpOutWorkThreadFp(void *arg) @@ -1090,34 +1225,34 @@ void* taosDumpOutWorkThreadFp(void *arg) STableRecord tableRecord; int fd; - char tmpFileName[TSDB_FILENAME_LEN*4] = {0}; - sprintf(tmpFileName, ".tables.tmp.%d", pThread->threadIndex); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + char tmpBuf[TSDB_FILENAME_LEN*4] = {0}; + sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); + fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd == -1) { - fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpFileName); + fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpBuf); return NULL; } FILE *fp = NULL; - memset(tmpFileName, 0, TSDB_FILENAME_LEN + 128); + memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); if (tsArguments.outpath[0] != 0) { - sprintf(tmpFileName, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); + sprintf(tmpBuf, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); } else { - sprintf(tmpFileName, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); + sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); } - fp = fopen(tmpFileName, "w"); + fp = fopen(tmpBuf, "w"); if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpFileName); + fprintf(stderr, "failed to open file %s\n", tmpBuf); close(fd); return NULL; } - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, "use %s", pThread->dbName); + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, "use %s", pThread->dbName); - TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpFileName); + TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf); int32_t code = taos_errno(tmpResult); if (code != 0) { fprintf(stderr, "invalid database %s\n", pThread->dbName); @@ -1127,11 +1262,47 @@ void* taosDumpOutWorkThreadFp(void *arg) return NULL; } + int fileNameIndex = 1; + int tablesInOneFile = 0; + int64_t lastRowsPrint = 5000000; fprintf(fp, "USE %s;\n\n", pThread->dbName); while (1) { ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); if (readLen <= 0) break; - taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon); + + int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon, pThread->dbName); + if (ret >= 0) { + // TODO: sum table count and table rows by self + pThread->tablesOfDumpOut++; + pThread->rowsOfDumpOut += ret; + + if (pThread->rowsOfDumpOut >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from database %s\n", pThread->rowsOfDumpOut, pThread->dbName); + lastRowsPrint += 5000000; + } + + tablesInOneFile++; + if (tablesInOneFile >= tsArguments.table_batch) { + fclose(fp); + tablesInOneFile = 0; + + memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); + if (tsArguments.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); + } else { + sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex); + } + fileNameIndex++; + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + fprintf(stderr, "failed to open file %s\n", tmpBuf); + close(fd); + taos_free_result(tmpResult); + return NULL; + } + } + } } taos_free_result(tmpResult); @@ -1141,21 +1312,18 @@ void* taosDumpOutWorkThreadFp(void *arg) return NULL; } -static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName) +static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName) { pthread_attr_t thattr; SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); for (int t = 0; t < numOfThread; ++t) { SThreadParaObj *pThread = threadObj + t; + pThread->rowsOfDumpOut = 0; + pThread->tablesOfDumpOut = 0; pThread->threadIndex = t; pThread->totalThreads = numOfThread; tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN); - pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); - - if (pThread->taosCon == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon)); - exit(0); - } + pThread->taosCon = taosCon; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -1170,15 +1338,24 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfTh pthread_join(threadObj[t].threadID, NULL); } + // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt + int64_t totalRowsOfDumpOut = 0; + int64_t totalChildTblsOfDumpOut = 0; for (int32_t t = 0; t < numOfThread; ++t) { - taos_close(threadObj[t].taosCon); + totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; + totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; } + + fprintf(g_fpOfResult, "# child table counter: %"PRId64"\n", totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# row counter: %"PRId64"\n", totalRowsOfDumpOut); + g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut; + g_resultStatistics.totalRowsOfDumpOut += totalRowsOfDumpOut; free(threadObj); } -int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) { +int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName) { int count = 0; STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); @@ -1187,15 +1364,15 @@ int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) { exit(-1); } - count = taosGetTableDes(table, tableDes, taosCon, true); + count = taosGetTableDes(dbName, table, tableDes, taosCon, true); if (count < 0) { free(tableDes); - fprintf(stderr, "failed to get stable schema\n"); + fprintf(stderr, "failed to get stable[%s] schema\n", table); exit(-1); } - taosDumpCreateTableClause(tableDes, count, fp); + taosDumpCreateTableClause(tableDes, count, fp, dbName); free(tableDes); return 0; @@ -1207,38 +1384,19 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) TAOS_ROW row; int fd = -1; STableRecord tableRecord; + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - exit(-1); - } - - sprintf(tmpCommand, "use %s", dbName); + sprintf(sqlstr, "show %s.stables", dbName); - TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(tmpResult); + TAOS_RES* res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); if (code != 0) { - fprintf(stderr, "invalid database %s, error: %s\n", dbName, taos_errstr(taosCon)); - free(tmpCommand); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason: %s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); exit(-1); } - - taos_free_result(tmpResult); - sprintf(tmpCommand, "show stables"); - - tmpResult = taos_query(taosCon, tmpCommand); - code = taos_errno(tmpResult); - if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", tmpCommand, taos_errstr(taosCon)); - free(tmpCommand); - taos_free_result(tmpResult); - exit(-1); - } - - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + TAOS_FIELD *fields = taos_fetch_fields(res); char tmpFileName[TSDB_FILENAME_LEN + 1]; memset(tmpFileName, 0, TSDB_FILENAME_LEN); @@ -1246,32 +1404,38 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd == -1) { fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(tmpResult); - free(tmpCommand); + taos_free_result(res); (void)remove(".stables.tmp"); exit(-1); } - while ((row = taos_fetch_row(tmpResult)) != NULL) { + while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); taosWrite(fd, &tableRecord, sizeof(STableRecord)); } - taos_free_result(tmpResult); + taos_free_result(res); (void)lseek(fd, 0, SEEK_SET); + int superTblCnt = 0; while (1) { ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); if (readLen <= 0) break; - (void)taosDumpStable(tableRecord.name, fp, taosCon); + int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName); + if (0 == ret) { + superTblCnt++; + } } + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + close(fd); (void)remove(".stables.tmp"); - free(tmpCommand); return 0; } @@ -1282,111 +1446,128 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao STableRecord tableRecord; taosDumpCreateDbClause(dbInfo, arguments->with_property, fp); - - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } - - sprintf(tmpCommand, "use %s", dbInfo->name); - TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(tmpResult); - if (code != 0) { - fprintf(stderr, "invalid database %s\n", dbInfo->name); - free(tmpCommand); - taos_free_result(tmpResult); - return -1; - } - taos_free_result(tmpResult); + fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfo->name); + g_resultStatistics.totalDatabasesOfDumpOut++; + + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; fprintf(fp, "USE %s;\n\n", dbInfo->name); (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); - sprintf(tmpCommand, "show tables"); + sprintf(sqlstr, "show %s.tables", dbInfo->name); - tmpResult = taos_query(taosCon, tmpCommand); - code = taos_errno(tmpResult); + TAOS_RES* res = taos_query(taosCon, sqlstr); + int code = taos_errno(res); if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tmpCommand); - free(tmpCommand); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); - - int32_t numOfTable = 0; - int32_t numOfThread = 0; - char tmpFileName[TSDB_FILENAME_LEN + 1]; - while ((row = taos_fetch_row(tmpResult)) != NULL) { - if (0 == numOfTable) { - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, ".tables.tmp.%d", numOfThread); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(tmpResult); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - (void)remove(tmpFileName); - } - free(tmpCommand); - return -1; - } + char tmpBuf[TSDB_FILENAME_LEN + 1]; + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".show-tables.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + taos_free_result(res); + return -1; + } - numOfThread++; - } + TAOS_FIELD *fields = taos_fetch_fields(res); + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); - + taosWrite(fd, &tableRecord, sizeof(STableRecord)); - + numOfTable++; + } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); + + int maxThreads = tsArguments.thread_num; + int tableOfPerFile ; + if (numOfTable <= tsArguments.thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / tsArguments.thread_num; + if (0 != numOfTable % tsArguments.thread_num) { + tableOfPerFile += 1; + } + } - if (numOfTable >= arguments->table_batch) { - numOfTable = 0; + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; + } + + int32_t numOfThread = 0; + int subFd = -1; + for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".show-tables.tmp"); + (void)remove(tmpBuf); + free(tblBuf); close(fd); - fd = -1; + return -1; + } + + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; } + taosWrite(subFd, tblBuf, readLen); + close(subFd); } + sprintf(tmpBuf, ".show-tables.tmp"); + (void)remove(tmpBuf); + if (fd >= 0) { close(fd); fd = -1; } - taos_free_result(tmpResult); + taos_free_result(res); // start multi threads to dumpout - taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name); + taosStartDumpOutWorkThreads(taosCon, arguments, numOfThread, dbInfo->name); for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - (void)remove(tmpFileName); - } - - free(tmpCommand); + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + free(tblBuf); return 0; } -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) { +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName) { int counter = 0; int count_temp = 0; + char sqlstr[COMMAND_SIZE]; - char* tmpBuf = (char *)malloc(COMMAND_SIZE); - if (tmpBuf == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return; - } - - char* pstr = tmpBuf; + char* pstr = sqlstr; - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s", tableDes->name); + pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", dbName, tableDes->name); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -1420,12 +1601,10 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) { pstr += sprintf(pstr, ");"); - fprintf(fp, "%s\n", tmpBuf); - - free(tmpBuf); + fprintf(fp, "%s\n\n", sqlstr); } -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp) { +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName) { int counter = 0; int count_temp = 0; @@ -1438,7 +1617,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols char *pstr = NULL; pstr = tmpBuf; - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s USING %s TAGS (", tableDes->name, metric); + pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", dbName, tableDes->name, dbName, metric); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -1479,48 +1658,36 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols free(tmpBuf); } -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon) { - /* char temp[MAX_COMMAND_SIZE] = "\0"; */ - int64_t totalRows = 0; +int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName) { + int64_t lastRowsPrint = 5000000; + int64_t totalRows = 0; int count = 0; char *pstr = NULL; TAOS_ROW row = NULL; int numFields = 0; - char *tbuf = NULL; - - char* tmpCommand = (char *)calloc(1, COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; + + if (arguments->schemaonly) { + return 0; } int32_t sql_buf_len = arguments->max_sql_len; char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128); if (tmpBuffer == NULL) { fprintf(stderr, "failed to allocate memory\n"); - free(tmpCommand); return -1; } pstr = tmpBuffer; - if (arguments->schemaonly) { - free(tmpCommand); - free(tmpBuffer); - return 0; - } + char sqlstr[1024] = {0}; + sprintf(sqlstr, + "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", + dbName, tbname, arguments->start_time, arguments->end_time); - sprintf(tmpCommand, - "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", - tbname, - arguments->start_time, - arguments->end_time); - - TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); + TAOS_RES* tmpResult = taos_query(taosCon, sqlstr); int32_t code = taos_errno(tmpResult); if (code != 0) { - fprintf(stderr, "failed to run command %s, reason: %s\n", tmpCommand, taos_errstr(taosCon)); - free(tmpCommand); + fprintf(stderr, "failed to run command %s, reason: %s\n", sqlstr, taos_errstr(tmpResult)); free(tmpBuffer); taos_free_result(tmpResult); return -1; @@ -1529,14 +1696,6 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* numFields = taos_field_count(tmpResult); assert(numFields > 0); TAOS_FIELD *fields = taos_fetch_fields(tmpResult); - tbuf = (char *)malloc(COMMAND_SIZE); - if (tbuf == NULL) { - fprintf(stderr, "No enough memory\n"); - free(tmpCommand); - free(tmpBuffer); - taos_free_result(tmpResult); - return -1; - } int rowFlag = 0; int32_t curr_sqlstr_len = 0; @@ -1550,7 +1709,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* if (count == 0) { total_sqlstr_len = 0; - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s VALUES (", tbname); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s.%s VALUES (", dbName, tbname); } else { if (arguments->mysqlFlag) { if (0 == rowFlag) { @@ -1594,17 +1753,21 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* case TSDB_DATA_TYPE_DOUBLE: curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col])); break; - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_BINARY: { + char tbuf[COMMAND_SIZE] = {0}; //*(pstr++) = '\''; converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); //pstr = stpcpy(pstr, tbuf); //*(pstr++) = '\''; pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; - case TSDB_DATA_TYPE_NCHAR: + } + case TSDB_DATA_TYPE_NCHAR: { + char tbuf[COMMAND_SIZE] = {0}; convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; + } case TSDB_DATA_TYPE_TIMESTAMP: if (!arguments->mysqlFlag) { curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", *(int64_t *)row[col]); @@ -1624,9 +1787,14 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ") "); - totalRows++; + totalRows++; count++; fprintf(fp, "%s", tmpBuffer); + + if (totalRows >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from %s.%s\n", totalRows, dbName, tbname); + lastRowsPrint += 5000000; + } total_sqlstr_len += curr_sqlstr_len; @@ -1638,19 +1806,12 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* //} } + fprintf(fp, "\n"); atomic_add_fetch_64(&totalDumpOutRows, totalRows); - - fprintf(fp, "\n"); - - if (tbuf) { - free(tbuf); - } taos_free_result(tmpResult); - tmpResult = NULL; - free(tmpCommand); free(tmpBuffer); - return 0; + return totalRows; } int taosCheckParam(struct arguments *arguments) { @@ -1950,13 +2111,15 @@ static void taosGetDirectoryFileList(char *inputDir) if (fileStat.st_mode & S_IFDIR) { taosCheckTablesSQLFile(inputDir); tsSqlFileNum = taosGetFilesNum(inputDir, "sql"); - int totalSQLFileNum = tsSqlFileNum; + int tsSqlFileNumOfTbls = tsSqlFileNum; if (tsDbSqlFile[0] != 0) { - tsSqlFileNum--; + tsSqlFileNumOfTbls--; } taosMallocSQLFiles(); - taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNum); - fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir); + if (0 != tsSqlFileNumOfTbls) { + taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNumOfTbls); + } + fprintf(stdout, "\nstart to dispose %d files in %s\n", tsSqlFileNum, inputDir); } else { fprintf(stderr, "ERROR: %s is not a directory\n", inputDir); @@ -1986,159 +2149,6 @@ static FILE* taosOpenDumpInFile(char *fptr) { return f; } -int taosDumpInOneFile_old(TAOS * taos, FILE* fp, char* fcharset, char* encode) { - char *command = NULL; - char *lcommand = NULL; - int tsize = 0; - char *line = NULL; - _Bool isRun = true; - size_t line_size = 0; - char *pstr = NULL; - char *lstr = NULL; - size_t inbytesleft = 0; - size_t outbytesleft = COMMAND_SIZE; - char *tcommand = NULL; - char *charsetOfFile = NULL; - iconv_t cd = (iconv_t)(-1); - - command = (char *)malloc(COMMAND_SIZE); - lcommand = (char *)malloc(COMMAND_SIZE); - if (command == NULL || lcommand == NULL) { - fprintf(stderr, "failed to connect to allocate memory\n"); - goto _dumpin_exit_failure; - } - - // Resolve locale - if (*fcharset != '\0') { - charsetOfFile = fcharset; - } else { - charsetOfFile = encode; - } - - if (charsetOfFile != NULL && strcasecmp(tsCharset, charsetOfFile) != 0) { - cd = iconv_open(tsCharset, charsetOfFile); - if (cd == ((iconv_t)(-1))) { - fprintf(stderr, "Failed to open iconv handle\n"); - goto _dumpin_exit_failure; - } - } - - pstr = command; - int64_t linenu = 0; - while (1) { - ssize_t size = getline(&line, &line_size, fp); - linenu++; - if (size <= 0) break; - if (size == 1) { - if (pstr != command) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != ((iconv_t)(-1))) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - - taosReplaceCtrlChar(tcommand); - - if (queryDB(taos, tcommand) != 0) { - fprintf(stderr, "error sql: linenu: %" PRId64 " failed\n", linenu); - exit(0); - } - - pstr = command; - pstr[0] = '\0'; - tsize = 0; - isRun = true; - } - - continue; - } - - /* if (line[0] == '-' && line[1] == '-') continue; */ - - line[size - 1] = 0; - - if (tsize + size - 1 > COMMAND_SIZE) { - fprintf(stderr, "command is too long\n"); - goto _dumpin_exit_failure; - } - - if (line[size - 2] == '\\') { - line[size - 2] = ' '; - isRun = false; - } else { - isRun = true; - } - - memcpy(pstr, line, size - 1); - pstr += (size - 1); - *pstr = '\0'; - - if (!isRun) continue; - - if (command != pstr && !isEmptyCommand(command)) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != ((iconv_t)(-1))) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - taosReplaceCtrlChar(tcommand); - if (queryDB(taos, tcommand) != 0) { - fprintf(stderr, "error sql: linenu:%" PRId64 " failed\n", linenu); - exit(0); - } - } - - pstr = command; - *pstr = '\0'; - tsize = 0; - } - - if (pstr != command) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != ((iconv_t)(-1))) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - taosReplaceCtrlChar(lcommand); - if (queryDB(taos, tcommand) != 0) - fprintf(stderr, "error sql: linenu:%" PRId64 " failed \n", linenu); - } - - if (cd != ((iconv_t)(-1))) iconv_close(cd); - tfree(line); - tfree(command); - tfree(lcommand); - taos_close(taos); - fclose(fp); - return 0; - -_dumpin_exit_failure: - if (cd != ((iconv_t)(-1))) iconv_close(cd); - tfree(command); - tfree(lcommand); - taos_close(taos); - fclose(fp); - return -1; -} - int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, char* fileName) { int read_len = 0; char * cmd = NULL; @@ -2152,6 +2162,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c return -1; } + int lastRowsPrint = 5000000; int lineNo = 0; while ((read_len = getline(&line, &line_len, fp)) != -1) { ++lineNo; @@ -2172,12 +2183,18 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c memcpy(cmd + cmd_len, line, read_len); cmd[read_len + cmd_len]= '\0'; - if (queryDB(taos, cmd)) { + if (queryDbImpl(taos, cmd)) { fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); + fprintf(g_fpOfResult, "error sql: linenu:%d, file:%s\n", lineNo, fileName); } memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); - cmd_len = 0; + cmd_len = 0; + + if (lineNo >= lastRowsPrint) { + printf(" %d lines already be executed from file %s\n", lineNo, fileName); + lastRowsPrint += 5000000; + } } tfree(cmd); @@ -2204,7 +2221,7 @@ void* taosDumpInWorkThreadFp(void *arg) return NULL; } -static void taosStartDumpInWorkThreads(struct arguments *args) +static void taosStartDumpInWorkThreads(void* taosCon, struct arguments *args) { pthread_attr_t thattr; SThreadParaObj *pThread; @@ -2219,11 +2236,7 @@ static void taosStartDumpInWorkThreads(struct arguments *args) pThread = threadObj + t; pThread->threadIndex = t; pThread->totalThreads = totalThreads; - pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); - if (pThread->taosCon == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon)); - exit(0); - } + pThread->taosCon = taosCon; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -2259,7 +2272,10 @@ int taosDumpIn(struct arguments *arguments) { taosGetDirectoryFileList(arguments->inpath); + int32_t tsSqlFileNumOfTbls = tsSqlFileNum; if (tsDbSqlFile[0] != 0) { + tsSqlFileNumOfTbls--; + fp = taosOpenDumpInFile(tsDbSqlFile); if (NULL == fp) { fprintf(stderr, "failed to open input file %s\n", tsDbSqlFile); @@ -2272,7 +2288,9 @@ int taosDumpIn(struct arguments *arguments) { taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile); } - taosStartDumpInWorkThreads(arguments); + if (0 != tsSqlFileNumOfTbls) { + taosStartDumpInWorkThreads(taos, arguments); + } taos_close(taos); taosFreeSQLFiles(); diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index ff5c9335b654edbdb7bbf6c081b38b265923d645..2df4708c239515febafc7a4f3ab3f63bd9e434e8 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -1,12 +1,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) +INCLUDE_DIRECTORIES(inc) +AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(mnode ${SRC}) -ENDIF () \ No newline at end of file +ADD_LIBRARY(mnode ${SRC}) diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index 9354b923d74de055d3905ebe6581d3f721630b08..d03ba8d717a13cc943f2494dc0dca0d59d108b6f 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -32,7 +32,7 @@ int32_t mnodeInitDbs(); void mnodeCleanupDbs(); int64_t mnodeGetDbNum(); SDbObj *mnodeGetDb(char *db); -SDbObj *mnodeGetDbByTableId(char *db); +SDbObj *mnodeGetDbByTableName(char *db); void * mnodeGetNextDb(void *pIter, SDbObj **pDb); void mnodeCancelGetNextDb(void *pIter); void mnodeIncDbRef(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 6d3061c4269f54f0075a07ee75115f43e856479b..5eeac97209634206d42e1c4aebec129e67ee38e5 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -128,8 +128,8 @@ typedef struct { typedef struct { int32_t dnodeId; int8_t role; - int8_t reserved[3]; - SDnodeObj* pDnode; + int8_t vver[3]; // To ensure compatibility, 3 bits are used to represent the remainder of 64 bit version + SDnodeObj *pDnode; } SVnodeGid; typedef struct SVgObj { @@ -138,13 +138,14 @@ typedef struct SVgObj { int64_t createdTime; int32_t lbDnodeId; int32_t lbTime; - char dbName[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + char dbName[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; int8_t inUse; int8_t accessState; int8_t status; int8_t reserved0[4]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; - int8_t reserved1[12]; + int32_t vgCfgVersion; + int8_t reserved1[8]; int8_t updateEnd[4]; int32_t refCount; int32_t numOfTables; @@ -173,15 +174,16 @@ typedef struct { int8_t replications; int8_t quorum; int8_t update; - int8_t reserved[11]; + int8_t cacheLastRow; + int8_t reserved[10]; } SDbCfg; typedef struct SDbObj { - char name[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + char name[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; int8_t reserved0[4]; char acct[TSDB_USER_LEN]; int64_t createdTime; - int32_t cfgVersion; + int32_t dbCfgVersion; SDbCfg cfg; int8_t status; int8_t reserved1[11]; @@ -257,7 +259,7 @@ typedef struct { int16_t bytes[TSDB_MAX_COLUMNS]; int32_t numOfReads; int8_t maxReplica; - int8_t reserved0[0]; + int8_t reserved0[1]; uint16_t payloadLen; char payload[]; } SShowObj; diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index 8bc29ef9efabf73470896b49b86aad276a29a748..fa1995254e9598614cebfef57221b3ff4238b6f1 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -52,18 +52,22 @@ typedef enum EDnodeOfflineReason { TAOS_DN_OFF_TIME_ZONE_NOT_MATCH, TAOS_DN_OFF_LOCALE_NOT_MATCH, TAOS_DN_OFF_CHARSET_NOT_MATCH, + TAOS_DN_OFF_FLOW_CTRL_NOT_MATCH, + TAOS_DN_OFF_SLAVE_QUERY_NOT_MATCH, + TAOS_DN_OFF_ADJUST_MASTER_NOT_MATCH, TAOS_DN_OFF_OTHERS } EDnodeOfflineReason; +extern char* dnodeStatus[]; +extern char* dnodeRoles[]; + int32_t mnodeInitDnodes(); void mnodeCleanupDnodes(); -char* mnodeGetDnodeStatusStr(int32_t dnodeStatus); -void mgmtMonitorDnodeModule(); - int32_t mnodeGetDnodesNum(); int32_t mnodeGetOnlinDnodesCpuCoreNum(); int32_t mnodeGetOnlineDnodesNum(); +void mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum); void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode); void mnodeCancelGetNextDnode(void *pIter); void mnodeIncDnodeRef(SDnodeObj *pDnode); diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index 93f2fa11ea95f22ca4addf12496b79cdaddbf290..ffdec02eb6ca3fcb2b6cd230fb93ebb26dea26de 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -43,8 +43,8 @@ void mnodeIncMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode); char * mnodeGetMnodeRoleStr(); -void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet); -void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect); +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect); char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 31ea2da640ef3b20b84ca769a22637332d6ffe34..e4e4a7a054ff194a578dbaad75b2ae17bfe6f1ad 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -59,8 +59,8 @@ typedef struct SSdbRow { SMnodeMsg *pMsg; int32_t (*fpReq)(SMnodeMsg *pMsg); int32_t (*fpRsp)(SMnodeMsg *pMsg, int32_t code); - char reserveForSync[16]; - SWalHead pHead[]; + char reserveForSync[24]; + SWalHead pHead; } SSdbRow; typedef struct { diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index ee9ec7ae9305c3d436b6e5bec0cd4e94ecc1adf4..2067ad04ccf963701bffcf134c8ae478547f1abb 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -53,6 +53,9 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromIp(char *ep); +int32_t mnodeGetVgidVer(int8_t *vver); +void mnodeSetVgidVer(int8_t *cver, uint64_t iver); + #ifdef __cplusplus } #endif diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 6fba05674fadbe967a673bbc0cc6e02ad7f2b41a..afe474df6b7d8d4bd1945523a95d839bcd29e247 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -81,7 +81,7 @@ static int32_t mnodeAcctActionDecode(SSdbRow *pRow) { } static int32_t mnodeAcctActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfRows = sdbGetNumOfRows(tsAcctSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root acct"); int32_t code = mnodeCreateRootAcct(); @@ -97,14 +97,14 @@ static int32_t mnodeAcctActionRestored() { int32_t mnodeInitAccts() { SAcctObj tObj; - tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsAcctUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_ACCOUNT, .name = "accounts", .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE, .maxRowSize = tsAcctUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeAcctActionInsert, .fpDelete = mnodeAcctActionDelete, @@ -206,7 +206,7 @@ void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { } static int32_t mnodeCreateRootAcct() { - int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); if (numOfAccts != 0) return TSDB_CODE_SUCCESS; SAcctObj *pAcct = malloc(sizeof(SAcctObj)); diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 56229daffa87d3300cc597d9353b5a720751a0f9..169d2ebd9dae1bcc34ea24dd3770951f9cbb442b 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -68,7 +68,7 @@ static int32_t mnodeClusterActionDecode(SSdbRow *pRow) { } static int32_t mnodeClusterActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfRows = sdbGetNumOfRows(tsClusterSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create cluster"); int32_t code = mnodeCreateCluster(); @@ -84,14 +84,14 @@ static int32_t mnodeClusterActionRestored() { int32_t mnodeInitCluster() { SClusterObj tObj; - tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsClusterUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_CLUSTER, .name = "cluster", .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE, .maxRowSize = tsClusterUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeClusterActionInsert, .fpDelete = mnodeClusterActionDelete, @@ -139,7 +139,7 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { } static int32_t mnodeCreateCluster() { - int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; SClusterObj *pCluster = malloc(sizeof(SClusterObj)); @@ -171,7 +171,7 @@ void mnodeUpdateClusterId() { void *pIter = mnodeGetNextCluster(NULL, &pCluster); if (pCluster != NULL) { tstrncpy(tsClusterId, pCluster->uid, TSDB_CLUSTER_ID_LEN); - mInfo("cluster id is set to %s", tsClusterId); + mDebug("cluster id is set to %s", tsClusterId); } mnodeDecClusterRef(pCluster); @@ -195,7 +195,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void * cols++; pMeta->numOfColumns = htons(cols); - strcpy(pMeta->tableId, "show cluster"); + strcpy(pMeta->tableFname, "show cluster"); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -226,7 +226,7 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *) pWrite = pCluster->createdTime; + *(int64_t *) pWrite = pCluster->createdTime; cols++; mnodeDecClusterRef(pCluster); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index c971a945aa37dac5530c5f08701e446d5ee79cea..9fdbaa79650c6b1a421501a7c5fe4b4d1709ea3f 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -143,14 +143,14 @@ static int32_t mnodeDbActionRestored() { int32_t mnodeInitDbs() { SDbObj tObj; - tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDbUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_DB, .name = "dbs", .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE, .maxRowSize = tsDbUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeDbActionInsert, .fpDelete = mnodeDbActionDelete, @@ -192,25 +192,20 @@ SDbObj *mnodeGetDb(char *db) { } void mnodeIncDbRef(SDbObj *pDb) { - return sdbIncRef(tsDbSdb, pDb); + sdbIncRef(tsDbSdb, pDb); } void mnodeDecDbRef(SDbObj *pDb) { - return sdbDecRef(tsDbSdb, pDb); + sdbDecRef(tsDbSdb, pDb); } -SDbObj *mnodeGetDbByTableId(char *tableId) { - char db[TSDB_TABLE_FNAME_LEN], *pos; - - // tableId format should be : acct.db.table - pos = strstr(tableId, TS_PATH_DELIMITER); - assert(NULL != pos); +SDbObj *mnodeGetDbByTableName(char *tableName) { + SName name = {0}; + tNameFromString(&name, tableName, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE); - pos = strstr(pos + 1, TS_PATH_DELIMITER); - assert(NULL != pos); - - memset(db, 0, sizeof(db)); - strncpy(db, tableId, pos - tableId); + // validate the tableName? + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); return mnodeGetDb(db); } @@ -311,24 +306,32 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { return TSDB_CODE_MND_INVALID_DB_OPTION; } - if (pCfg->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCfg->quorum > TSDB_MAX_DB_REPLICA_OPTION) { - mError("invalid db option quorum:%d valid range: [%d, %d]", pCfg->quorum, TSDB_MIN_DB_REPLICA_OPTION, - TSDB_MAX_DB_REPLICA_OPTION); + if (pCfg->replications > mnodeGetDnodesNum()) { + mError("no enough dnode to config replica: %d, #dnodes: %d", pCfg->replications, mnodeGetDnodesNum()); return TSDB_CODE_MND_INVALID_DB_OPTION; } -#ifndef _SYNC - if (pCfg->replications != 1) { - mError("invalid db option replications:%d can only be 1 in this version", pCfg->replications); + if (pCfg->quorum > pCfg->replications) { + mError("invalid db option quorum:%d larger than replica:%d", pCfg->quorum, pCfg->replications); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + + if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { + mError("invalid db option quorum:%d valid range: [%d, %d]", pCfg->quorum, TSDB_MIN_DB_QUORUM_OPTION, + TSDB_MAX_DB_QUORUM_OPTION); return TSDB_CODE_MND_INVALID_DB_OPTION; } -#endif if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE); return TSDB_CODE_MND_INVALID_DB_OPTION; } + if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { + mError("invalid db option cacheLastRow:%d valid range: [%d, %d]", pCfg->cacheLastRow, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + return TSDB_CODE_SUCCESS; } @@ -350,6 +353,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->replications < 0) pCfg->replications = tsReplications; if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; if (pCfg->update < 0) pCfg->update = tsUpdate; + if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; } static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { @@ -403,7 +407,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * .walLevel = pCreate->walLevel, .replications = pCreate->replications, .quorum = pCreate->quorum, - .update = pCreate->update + .update = pCreate->update, + .cacheLastRow = pCreate->cacheLastRow }; mnodeSetDefaultDbCfg(&pDb->cfg); @@ -563,7 +568,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "keep1,keep2,keep(D)"); + strcpy(pSchema[cols].name, "keep0,keep1,keep(D)"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -612,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn strcpy(pSchema[cols].name, "comp"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "cachelast"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; #ifndef __CLOUD_VERSION__ } #endif @@ -668,7 +679,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb); if (pDb == NULL) break; - if (pDb->pAcct != pUser->pAcct) { + if (pDb->pAcct != pUser->pAcct || pDb->status != TSDB_DB_STATUS_READY) { mnodeDecDbRef(pDb); continue; } @@ -757,6 +768,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int8_t *)pWrite = pDb->cfg.compression; cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.cacheLastRow; + cols++; #ifndef __CLOUD_VERSION__ } #endif @@ -871,6 +886,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { int8_t quorum = pAlter->quorum; int8_t precision = pAlter->precision; int8_t update = pAlter->update; + int8_t cacheLastRow = pAlter->cacheLastRow; terrno = TSDB_CODE_SUCCESS; @@ -983,6 +999,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { #endif } + if (cacheLastRow >= 0 && cacheLastRow != pDb->cfg.cacheLastRow) { + mDebug("db:%s, cacheLastRow:%d change to %d", pDb->name, pDb->cfg.cacheLastRow, cacheLastRow); + newCfg.cacheLastRow = cacheLastRow; + } + return newCfg; } @@ -1022,7 +1043,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; - pDb->cfgVersion++; + pDb->dbCfgVersion++; SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 0ff50b23072b11ab8cc31b192ea51a3c1c3004ad..304096f3ae6178bdb5b7ce0620d81cf29bef11d0 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,7 +39,7 @@ #include "mnodeCluster.h" int32_t tsAccessSquence = 0; -int64_t tsDnodeRid = -1; +int64_t tsDnodeRid = -1; static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; @@ -63,7 +63,6 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole); static void mnodeUpdateDnodeEps(); static char* offlineReason[] = { @@ -104,16 +103,13 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) { dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort); mnodeUpdateDnodeEps(); - mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); + mInfo("dnode:%d, fqdn:%s ep:%s port:%d is created", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); return TSDB_CODE_SUCCESS; } static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) { SDnodeObj *pDnode = pRow->pObj; -#ifndef _SYNC - mnodeDropAllDnodeVgroups(pDnode); -#endif mnodeDropMnodeLocal(pDnode->dnodeId); bnNotify(); mnodeUpdateDnodeEps(); @@ -152,7 +148,7 @@ static int32_t mnodeDnodeActionDecode(SSdbRow *pRow) { } static int32_t mnodeDnodeActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); + int64_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create dnode:%s", tsLocalEp); mnodeCreateDnode(tsLocalEp, NULL); @@ -169,7 +165,7 @@ static int32_t mnodeDnodeActionRestored() { int32_t mnodeInitDnodes() { SDnodeObj tObj; - tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); pthread_mutex_init(&tsDnodeEpsMutex, NULL); SSdbTableDesc desc = { @@ -177,7 +173,7 @@ int32_t mnodeInitDnodes() { .name = "dnodes", .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE, .maxRowSize = tsDnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeDnodeActionInsert, .fpDelete = mnodeDnodeActionDelete, @@ -231,7 +227,7 @@ void mnodeCancelGetNextDnode(void *pIter) { } int32_t mnodeGetDnodesNum() { - return sdbGetNumOfRows(tsDnodeSdb); + return (int32_t)sdbGetNumOfRows(tsDnodeSdb); } int32_t mnodeGetOnlinDnodesCpuCoreNum() { @@ -267,6 +263,28 @@ int32_t mnodeGetOnlineDnodesNum() { return onlineDnodes; } +void mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum) { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + int32_t onlineDnodes = 0, totalDnodes = 0; + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + if (pDnode->status != TAOS_DN_STATUS_OFFLINE) ++onlineDnodes; + ++totalDnodes; + mnodeDecDnodeRef(pDnode); + } + + if (onlineNum) { + *onlineNum = onlineDnodes; + } + + if (totalNum) { + *totalNum = totalDnodes; + } +} + void *mnodeGetDnode(int32_t dnodeId) { return sdbGetRow(tsDnodeSdb, &dnodeId); } @@ -306,7 +324,7 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) { int32_t code = sdbUpdateRow(&row); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mError("dnodeId:%d, failed update", pDnode->dnodeId); + mError("dnode:%d, failed update", pDnode->dnodeId); } } @@ -379,10 +397,6 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { mError("\"numOfMnodes\"[%d - %d] cfg parameters inconsistent", clusterCfg->numOfMnodes, htonl(tsNumOfMnodes)); return TAOS_DN_OFF_NUM_OF_MNODES_NOT_MATCH; } - if (clusterCfg->enableBalance != htonl(tsEnableBalance)) { - mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, htonl(tsEnableBalance)); - return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH; - } if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) { mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg parameters inconsistent", clusterCfg->mnodeEqualVnodeNum, htonl(tsMnodeEqualVnodeNum)); @@ -415,7 +429,7 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { int64_t checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) { mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg parameters inconsistent", clusterCfg->timezone, @@ -432,6 +446,23 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { return TAOS_DN_OFF_CHARSET_NOT_MATCH; } + if (clusterCfg->enableBalance != tsEnableBalance) { + mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, tsEnableBalance); + return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH; + } + if (clusterCfg->flowCtrl != tsEnableFlowCtrl) { + mError("\"flowCtrl\"[%d - %d] cfg parameters inconsistent", clusterCfg->flowCtrl, tsEnableFlowCtrl); + return TAOS_DN_OFF_FLOW_CTRL_NOT_MATCH; + } + if (clusterCfg->slaveQuery != tsEnableSlaveQuery) { + mError("\"slaveQuery\"[%d - %d] cfg parameters inconsistent", clusterCfg->slaveQuery, tsEnableSlaveQuery); + return TAOS_DN_OFF_SLAVE_QUERY_NOT_MATCH; + } + if (clusterCfg->adjustMaster != tsEnableAdjustMaster) { + mError("\"adjustMaster\"[%d - %d] cfg parameters inconsistent", clusterCfg->adjustMaster, tsEnableAdjustMaster); + return TAOS_DN_OFF_ADJUST_MASTER_NOT_MATCH; + } + return 0; } @@ -560,7 +591,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { for (int32_t j = 0; j < openVnodes; ++j) { SVnodeLoad *pVload = &pStatus->load[j]; pVload->vgId = htonl(pVload->vgId); - pVload->cfgVersion = htonl(pVload->cfgVersion); + pVload->dbCfgVersion = htonl(pVload->dbCfgVersion); + pVload->vgCfgVersion = htonl(pVload->vgCfgVersion); + pVload->vnodeVersion = htobe64(pVload->vnodeVersion); SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId); if (pVgroup == NULL) { @@ -627,9 +660,9 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { char *temp = strchr(dnodeEp, ':'); if (!temp) { - int len = strlen(dnodeEp); + int32_t len = (int32_t)strlen(dnodeEp); if (dnodeEp[len - 1] == ';') dnodeEp[len - 1] = 0; - len = strlen(dnodeEp); + len = (int32_t)strlen(dnodeEp); snprintf(dnodeEp + len, TSDB_EP_LEN - len, ":%d", tsServerPort); } ep = dnodeEp; @@ -705,11 +738,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { mInfo("dnode:%d, start to drop it", pDnode->dnodeId); -#ifndef _SYNC - int32_t code = mnodeDropDnode(pDnode, pMsg); -#else int32_t code = bnDropDnode(pDnode); -#endif mnodeDecDnodeRef(pDnode); return code; } @@ -803,6 +832,10 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC } pShow->numOfRows = mnodeGetDnodesNum(); + if (tsArbitrator[0] != 0) { + pShow->numOfRows++; + } + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pIter = NULL; @@ -814,7 +847,7 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; int32_t cols = 0; - SDnodeObj *pDnode = NULL; + SDnodeObj *pDnode = NULL; char *pWrite; while (numOfRows < rows) { @@ -840,12 +873,12 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char* status = mnodeGetDnodeStatusStr(pDnode->status); + char* status = dnodeStatus[pDnode->status]; STR_TO_VARSTR(pWrite, status); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char* role = mnodeGetDnodeAlternativeRoleStr(pDnode->alternativeRole); + char* role = dnodeRoles[pDnode->alternativeRole]; STR_TO_VARSTR(pWrite, role); cols++; @@ -857,10 +890,49 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); cols++; - numOfRows++; + numOfRows++; mnodeDecDnodeRef(pDnode); } + if (tsArbitrator[0] != 0) { + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tsArbitrator, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *status = dnodeStatus[tsArbOnline > 0 ? TAOS_DN_STATUS_READY : TAOS_DN_STATUS_OFFLINE]; + STR_TO_VARSTR(pWrite, status); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, "arb"); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, "-"); + cols++; + + numOfRows++; + } + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; @@ -1038,6 +1110,11 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int8_t *)cfg->ptr)); + varDataSetLen(pWrite, t); + numOfRows++; + break; case TAOS_CFG_VTYPE_INT16: t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); varDataSetLen(pWrite, t); @@ -1161,76 +1238,17 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } -char* mnodeGetDnodeStatusStr(int32_t dnodeStatus) { - switch (dnodeStatus) { - case TAOS_DN_STATUS_OFFLINE: return "offline"; - case TAOS_DN_STATUS_DROPPING: return "dropping"; - case TAOS_DN_STATUS_BALANCING: return "balancing"; - case TAOS_DN_STATUS_READY: return "ready"; - default: return "undefined"; - } -} - -static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) { - switch (alternativeRole) { - case TAOS_DN_ALTERNATIVE_ROLE_ANY: return "any"; - case TAOS_DN_ALTERNATIVE_ROLE_MNODE: return "mnode"; - case TAOS_DN_ALTERNATIVE_ROLE_VNODE: return "vnode"; - default:return "any"; - } -} - -#ifndef _SYNC - -int32_t bnInit() { return TSDB_CODE_SUCCESS; } -void bnCleanUp() {} -void bnNotify() {} -void bnCheckModules() {} -void bnReset() {} -int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; } - -char* syncRole[] = { +char* dnodeStatus[] = { "offline", - "unsynced", - "syncing", - "slave", - "master" + "dropping", + "balancing", + "ready", + "undefined" }; -int32_t bnAllocVnodes(SVgObj *pVgroup) { - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - SDnodeObj *pSelDnode = NULL; - float vnodeUsage = 1000.0; - - while (1) { - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - - if (pDnode->numOfCores > 0 && pDnode->openVnodes < TSDB_MAX_VNODES) { - float openVnodes = pDnode->openVnodes; - if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum; - - float usage = openVnodes / pDnode->numOfCores; - if (usage <= vnodeUsage) { - pSelDnode = pDnode; - vnodeUsage = usage; - } - } - mnodeDecDnodeRef(pDnode); - } - - if (pSelDnode == NULL) { - mError("failed to alloc vnode to vgroup"); - return TSDB_CODE_MND_NO_ENOUGH_DNODES; - } - - pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId; - pVgroup->vnodeGid[0].pDnode = pSelDnode; - - mDebug("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes); - return TSDB_CODE_SUCCESS; -} - -#endif - +char* dnodeRoles[] = { + "any", + "mnode", + "vnode", + "any" +}; diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 86f2c821f9c4d088f8c057dacbf9c43b20270a90..7ef0488c420dd470c3afc5d7ca8ac7a518ccdc73 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -37,16 +37,10 @@ #include "mnodeShow.h" #include "mnodeProfile.h" -typedef struct { - const char *const name; - int (*init)(); - void (*cleanup)(); -} SMnodeComponent; - void *tsMnodeTmr = NULL; static bool tsMgmtIsRunning = false; -static const SMnodeComponent tsMnodeComponents[] = { +static SStep tsMnodeSteps[] = { {"sdbref", sdbInitRef, sdbCleanUpRef}, {"profile", mnodeInitProfile, mnodeCleanupProfile}, {"cluster", mnodeInitCluster, mnodeCleanupCluster}, @@ -67,34 +61,26 @@ static void mnodeInitTimer(); static void mnodeCleanupTimer(); static bool mnodeNeedStart() ; -static void mnodeCleanupComponents(int32_t stepId) { - for (int32_t i = stepId; i >= 0; i--) { - tsMnodeComponents[i].cleanup(); - } +static void mnodeCleanupComponents() { + int32_t stepSize = sizeof(tsMnodeSteps) / sizeof(SStep); + dnodeStepCleanup(tsMnodeSteps, stepSize); } static int32_t mnodeInitComponents() { - int32_t code = 0; - for (int32_t i = 0; i < sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]); i++) { - if (tsMnodeComponents[i].init() != 0) { - mnodeCleanupComponents(i); - code = -1; - break; - } - } - return code; + int32_t stepSize = sizeof(tsMnodeSteps) / sizeof(SStep); + return dnodeStepInit(tsMnodeSteps, stepSize); } int32_t mnodeStartSystem() { if (tsMgmtIsRunning) { mInfo("mnode module already started..."); - return 0; + return TSDB_CODE_SUCCESS; } mInfo("starting to initialize mnode ..."); if (mkdir(tsMnodeDir, 0755) != 0 && errno != EEXIST) { mError("failed to init mnode dir:%s, reason:%s", tsMnodeDir, strerror(errno)); - return -1; + return TSDB_CODE_MND_FAILED_TO_CREATE_DIR; } dnodeAllocMWritequeue(); @@ -102,9 +88,10 @@ int32_t mnodeStartSystem() { dnodeAllocateMPeerQueue(); if (mnodeInitComponents() != 0) { - return -1; + return TSDB_CODE_MND_FAILED_TO_INIT_STEP; } + dnodeReportStep("mnode-grant", "start to set grant infomation", 0); grantReset(TSDB_GRANT_ALL, 0); tsMgmtIsRunning = true; @@ -112,7 +99,7 @@ int32_t mnodeStartSystem() { sdbUpdateSync(NULL); - return 0; + return TSDB_CODE_SUCCESS; } int32_t mnodeInitSystem() { @@ -132,7 +119,7 @@ void mnodeCleanupSystem() { dnodeFreeMReadQueue(); dnodeFreeMPeerQueue(); mnodeCleanupTimer(); - mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1); + mnodeCleanupComponents(); mInfo("mnode is cleaned up"); } diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 68acae7dec0e5505e72733a638abcc142eeeb407..49473d3e06f54e1e66f507957d6351162990e030 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -72,7 +72,7 @@ static int32_t mnodeMnodeActionInsert(SSdbRow *pRow) { pDnode->isMgmt = true; mnodeDecDnodeRef(pDnode); - mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, + mInfo("mnode:%d, fqdn:%s ep:%s port:%u is created", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); return TSDB_CODE_SUCCESS; } @@ -136,14 +136,14 @@ int32_t mnodeInitMnodes() { mnodeMnodeInitLock(); SMnodeObj tObj; - tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsMnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_MNODE, .name = "mnodes", .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE, .maxRowSize = tsMnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_INT, .fpInsert = mnodeMnodeActionInsert, .fpDelete = mnodeMnodeActionDelete, @@ -176,7 +176,7 @@ void mnodeCleanupMnodes() { } int32_t mnodeGetMnodesNum() { - return sdbGetNumOfRows(tsMnodeSdb); + return (int32_t)sdbGetNumOfRows(tsMnodeSdb); } void *mnodeGetMnode(int32_t mnodeId) { @@ -202,13 +202,13 @@ void mnodeCancelGetNextMnode(void *pIter) { void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { bool set = false; SMInfos mInfos = {0}; - mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); if (pMinfos != NULL) { + mInfo("vgId:1, update mnodes epSet, numOfMinfos:%d", pMinfos->mnodeNum); set = true; mInfos = *pMinfos; - } - else { + } else { + mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d", mnodeGetMnodesNum()); int32_t index = 0; void * pIter = NULL; while (1) { @@ -273,14 +273,14 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { mnodeMnodeUnLock(); } -void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect) { mnodeMnodeRdLock(); *epSet = tsMEpForPeer; mnodeMnodeUnLock(); mTrace("vgId:1, mnodes epSet for peer is returned, num:%d inUse:%d", tsMEpForPeer.numOfEps, tsMEpForPeer.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { - if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { + if (redirect && strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { epSet->inUse = (i + 1) % epSet->numOfEps; mTrace("vgId:1, mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { @@ -289,14 +289,19 @@ void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { } } -void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect) { mnodeMnodeRdLock(); *epSet = tsMEpForShell; mnodeMnodeUnLock(); + if (mnodeGetDnodesNum() <= 1) { + epSet->numOfEps = 0; + return; + } + mTrace("vgId:1, mnodes epSet for shell is returned, num:%d inUse:%d", tsMEpForShell.numOfEps, tsMEpForShell.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { - if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { + if (redirect && strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { epSet->inUse = (i + 1) % epSet->numOfEps; mTrace("vgId:1, mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { @@ -372,6 +377,25 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) { return code; } +static bool mnodeAllOnline() { + void *pIter = NULL; + bool allOnline = true; + + while (1) { + SMnodeObj *pMnode = NULL; + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + if (pMnode->role != TAOS_SYNC_ROLE_MASTER && pMnode->role != TAOS_SYNC_ROLE_SLAVE) { + allOnline = false; + mDebug("mnode:%d, role:%s, not online", pMnode->mnodeId, syncRole[pMnode->role]); + mnodeDecMnodeRef(pMnode); + } + } + mnodeCancelGetNextMnode(pIter); + + return allOnline; +} + void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj)); pMnode->mnodeId = dnodeId; @@ -384,6 +408,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { .fpRsp = mnodeCreateMnodeCb }; + if (needConfirm && !mnodeAllOnline()) { + mDebug("wait all mnode online then create new mnode"); + return; + } + int32_t code = TSDB_CODE_SUCCESS; if (needConfirm) { code = mnodeSendCreateMnodeMsg(dnodeId, dnodeEp); diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index cfb7b7781b04c0145f66a58c6d3d2eaefa0913f3..9bd8d7e4d7b0723ac4c5e4248e3c1193dfc15ceb 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -17,7 +17,6 @@ #include "os.h" #include "taoserror.h" #include "tsched.h" -#include "tsystem.h" #include "tutil.h" #include "tgrant.h" #include "tbn.h" @@ -54,7 +53,7 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); - mnodeGetMnodeEpSetForPeer(epSet); + mnodeGetMnodeEpSetForPeer(epSet, true); rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 36b6ff7a59fc1f1c73e64c40ff3a5a1cef65ece6..17a4282d05935684df4ab6585fef5f2398a62979 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -282,27 +282,34 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi // not thread safe, need optimized int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) { - pConn->numOfQueries = htonl(pHBMsg->numOfQueries); - if (pConn->numOfQueries > 0) { + pConn->numOfQueries = 0; + pConn->numOfStreams = 0; + int32_t numOfQueries = htonl(pHBMsg->numOfQueries); + int32_t numOfStreams = htonl(pHBMsg->numOfStreams); + + if (numOfQueries > 0) { if (pConn->pQueries == NULL) { pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE); } - int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfQueries) * sizeof(SQueryDesc); + pConn->numOfQueries = MIN(QUERY_STREAM_SAVE_SIZE, numOfQueries); + + int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); if (saveSize > 0 && pConn->pQueries != NULL) { memcpy(pConn->pQueries, pHBMsg->pData, saveSize); } } - pConn->numOfStreams = htonl(pHBMsg->numOfStreams); - if (pConn->numOfStreams > 0) { + if (numOfStreams > 0) { if (pConn->pStreams == NULL) { pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE); } - int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfStreams) * sizeof(SStreamDesc); + pConn->numOfStreams = MIN(QUERY_STREAM_SAVE_SIZE, numOfStreams); + + int32_t saveSize = pConn->numOfStreams * sizeof(SStreamDesc); if (saveSize > 0 && pConn->pStreams != NULL) { - memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc), saveSize); + memcpy(pConn->pStreams, pHBMsg->pData + numOfQueries * sizeof(SQueryDesc), saveSize); } } @@ -349,7 +356,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; - strcpy(pSchema[cols].name, "time(us)"); + strcpy(pSchema[cols].name, "time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -378,12 +385,22 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v SConnObj *pConnObj = NULL; int32_t cols = 0; char * pWrite; + void * pIter; char str[TSDB_IPv4ADDR_LEN + 6] = {0}; while (numOfRows < rows) { - pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); - if (pConnObj == NULL) break; + pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + + if (numOfRows + pConnObj->numOfQueries >= rows) { + mnodeCancelGetNextConn(pIter); + break; + } + pShow->pIter = pIter; for (int32_t i = 0; i < pConnObj->numOfQueries; ++i) { SQueryDesc *pDesc = pConnObj->pQueries + i; cols = 0; @@ -450,6 +467,12 @@ static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "dest table"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); @@ -505,12 +528,22 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v SConnObj *pConnObj = NULL; int32_t cols = 0; char * pWrite; + void * pIter; char ipStr[TSDB_IPv4ADDR_LEN + 6]; while (numOfRows < rows) { - pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); - if (pConnObj == NULL) break; + pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + if (numOfRows + pConnObj->numOfStreams >= rows) { + mnodeCancelGetNextConn(pIter); + break; + } + + pShow->pIter = pIter; for (int32_t i = 0; i < pConnObj->numOfStreams; ++i) { SStreamDesc *pDesc = pConnObj->pStreams + i; cols = 0; @@ -524,6 +557,10 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->dstTable, pShow->bytes[cols]); + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index c2a70bc01d2194fff5b8095d06e40535b940dc25..dadd8155d8648244dba762afec63bfc6f2ac3721 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -50,7 +50,10 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); - mnodeGetMnodeEpSetForShell(epSet); + if (!epSet) { + return TSDB_CODE_MND_OUT_OF_MEMORY; + } + mnodeGetMnodeEpSetForShell(epSet, true); rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6cc4e097350783a93300c46adbd5f68faf43e143..fe1f70cb50e4dd074962d42ea475095c37667c38 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -183,18 +183,23 @@ static int32_t sdbInitWal() { return -1; } - sdbInfo("vgId:1, open wal for restore"); + sdbInfo("vgId:1, open sdb wal for restore"); int32_t code = walRestore(tsSdbMgmt.wal, NULL, sdbProcessWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, failed to open wal for restore since %s", tstrerror(code)); return -1; } + + sdbInfo("vgId:1, sdb wal load success"); return 0; } static void sdbRestoreTables() { int32_t totalRows = 0; int32_t numOfTables = 0; + + sdbInfo("vgId:1, sdb start to check for integrity"); + for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) { SSdbTable *pTable = sdbGetTableFromId(tableId); if (pTable == NULL) continue; @@ -202,9 +207,9 @@ static void sdbRestoreTables() { (*pTable->fpRestored)(); } - totalRows += pTable->numOfRows; + totalRows += (int32_t)pTable->numOfRows; numOfTables++; - sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->name, pTable->numOfRows); + sdbInfo("vgId:1, sdb:%s is checked, rows:%" PRId64, pTable->name, pTable->numOfRows); } sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbMgmt.version, totalRows, numOfTables); @@ -220,6 +225,10 @@ void sdbUpdateMnodeRoles() { for (int32_t i = 0; i < tsSdbMgmt.cfg.replica; ++i) { SMnodeObj *pMnode = mnodeGetMnode(roles.nodeId[i]); if (pMnode != NULL) { + if (pMnode->role != roles.role[i]) { + bnNotify(); + } + pMnode->role = roles.role[i]; sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]); if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role; @@ -233,11 +242,6 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeEpSet(NULL); } -static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { - sdbUpdateMnodeRoles(); - return 0; -} - static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { return walGetWalFile(tsSdbMgmt.wal, fileName, fileId); } @@ -253,7 +257,9 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) { sdbUpdateMnodeRoles(); } -static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; } +static void sdbStartFileSync(int32_t vgId) {} + +static void sdbStopFileSync(int32_t vgId, uint64_t fversion) {} static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {} @@ -265,7 +271,7 @@ static int32_t sdbGetSyncVersion(int32_t vgId, uint64_t *fver, uint64_t *vver) { // failed to forward, need revert insert static void sdbHandleFailedConfirm(SSdbRow *pRow) { - SWalHead *pHead = pRow->pHead; + SWalHead *pHead = &pRow->pHead; int32_t action = pHead->msgType % 10; sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pRow->pObj, @@ -309,11 +315,11 @@ void sdbUpdateAsync() { taosTmrReset(sdbUpdateSyncTmrFp, 200, NULL, tsMnodeTmr, &tsSdbTmr); } -void sdbUpdateSync(void *pMnodes) { +int32_t sdbUpdateSync(void *pMnodes) { SMInfos *pMinfos = pMnodes; if (!mnodeIsRunning()) { mDebug("vgId:1, mnode not start yet, update sync config later"); - return; + return TSDB_CODE_MND_MNODE_IS_RUNNING; } mDebug("vgId:1, update sync config, pMnodes:%p", pMnodes); @@ -368,12 +374,12 @@ void sdbUpdateSync(void *pMnodes) { if (!hasThisDnode) { sdbDebug("vgId:1, update sync config, this dnode not exist"); - return; + return TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC; } if (memcmp(&syncCfg, &tsSdbMgmt.cfg, sizeof(SSyncCfg)) == 0) { sdbDebug("vgId:1, update sync config, info not changed"); - return; + return TSDB_CODE_SUCCESS; } sdbInfo("vgId:1, work as mnode, replica:%d", syncCfg.replica); @@ -387,23 +393,26 @@ void sdbUpdateSync(void *pMnodes) { syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; sprintf(syncInfo.path, "%s", tsMnodeDir); - syncInfo.getFileInfo = sdbGetFileInfo; - syncInfo.getWalInfo = sdbGetWalInfo; - syncInfo.writeToCache = sdbWriteFwdToQueue; + syncInfo.getWalInfoFp = sdbGetWalInfo; + syncInfo.writeToCacheFp = sdbWriteFwdToQueue; syncInfo.confirmForward = sdbConfirmForward; - syncInfo.notifyRole = sdbNotifyRole; - syncInfo.notifyFileSynced = sdbNotifyFileSynced; - syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl; - syncInfo.getVersion = sdbGetSyncVersion; + syncInfo.notifyRoleFp = sdbNotifyRole; + syncInfo.startSyncFileFp = sdbStartFileSync; + syncInfo.stopSyncFileFp = sdbStopFileSync; + syncInfo.notifyFlowCtrlFp = sdbNotifyFlowCtrl; + syncInfo.getVersionFp = sdbGetSyncVersion; tsSdbMgmt.cfg = syncCfg; if (tsSdbMgmt.sync) { - syncReconfig(tsSdbMgmt.sync, &syncCfg); + int32_t code = syncReconfig(tsSdbMgmt.sync, &syncCfg); + if (code != 0) return code; } else { tsSdbMgmt.sync = syncStart(&syncInfo); + if (tsSdbMgmt.sync <= 0) return TSDB_CODE_MND_FAILED_TO_START_SYNC; } sdbUpdateMnodeRoles(); + return TSDB_CODE_SUCCESS; } int32_t sdbInitRef() { @@ -463,7 +472,7 @@ void sdbIncRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); } @@ -472,11 +481,11 @@ void sdbDecRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char *)pRow + pTable->refCountPos - 4); if (refCount <= 0 && *updateEnd) { sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); SSdbRow row = {.pObj = pRow}; @@ -489,7 +498,7 @@ static void *sdbGetRowMeta(SSdbTable *pTable, void *key) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } void **ppRow = (void **)taosHashGet(pTable->iHandle, key, keySize); @@ -522,7 +531,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -552,7 +561,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { } static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { - int32_t *updateEnd = pRow->pObj + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow->pObj + pTable->refCountPos - 4); bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name, @@ -565,7 +574,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { void * key = sdbGetObjKey(pTable, pRow->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -628,6 +637,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * SSdbTable *pTable = sdbGetTableFromId(tableId); assert(pTable != NULL); + if (!mnodeIsRunning() && tsSdbMgmt.version % 100000 == 0) { + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "%" PRIu64 " rows have been restored", tsSdbMgmt.version); + dnodeReportStep("mnode-sdb", stepDesc, 0); + } + if (qtype == TAOS_QTYPE_QUERY) return sdbPerformDeleteAction(pHead, pTable); pthread_mutex_lock(&tsSdbMgmt.mutex); @@ -746,7 +761,7 @@ bool sdbCheckRowDeleted(void *tparam, void *pRow) { SSdbTable *pTable = tparam; if (pTable == NULL) return false; - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow + pTable->refCountPos - 4); return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; } @@ -924,14 +939,14 @@ static int32_t sdbInitWorker() { static void sdbCleanupWorker() { for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsSdbWQset); } } for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } @@ -994,7 +1009,7 @@ static void sdbFreeQueue() { } static int32_t sdbWriteToQueue(SSdbRow *pRow, int32_t qtype) { - SWalHead *pHead = pRow->pHead; + SWalHead *pHead = &pRow->pHead; if (pHead->len > TSDB_MAX_WAL_SIZE) { sdbError("vgId:1, wal len:%d exceeds limit, hver:%" PRIu64, pHead->len, pHead->version); @@ -1033,10 +1048,13 @@ static int32_t sdbWriteFwdToQueue(int32_t vgId, void *wparam, int32_t qtype, voi return TSDB_CODE_VND_OUT_OF_MEMORY; } - memcpy(pRow->pHead, pHead, sizeof(SWalHead) + pHead->len); - pRow->rowData = pRow->pHead->cont; + memcpy(&pRow->pHead, pHead, sizeof(SWalHead) + pHead->len); + pRow->rowData = pRow->pHead.cont; - return sdbWriteToQueue(pRow, qtype); + int32_t code = sdbWriteToQueue(pRow, qtype); + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) code = 0; + + return code; } static int32_t sdbWriteRowToQueue(SSdbRow *pInputRow, int32_t action) { @@ -1052,7 +1070,7 @@ static int32_t sdbWriteRowToQueue(SSdbRow *pInputRow, int32_t action) { memcpy(pRow, pInputRow, sizeof(SSdbRow)); pRow->processedCount = 1; - SWalHead *pHead = pRow->pHead; + SWalHead *pHead = &pRow->pHead; pRow->rowData = pHead->cont; (*pTable->fpEncode)(pRow); @@ -1070,6 +1088,8 @@ static void *sdbWorkerFp(void *pWorker) { int32_t qtype; void * unUsed; + taosBlockSIGPIPE(); + while (1) { int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); if (numOfMsgs == 0) { @@ -1080,9 +1100,9 @@ static void *sdbWorkerFp(void *pWorker) { for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(tsSdbWQall, &qtype, (void **)&pRow); sdbTrace("vgId:1, msg:%p, row:%p hver:%" PRIu64 ", will be processed in sdb queue", pRow->pMsg, pRow->pObj, - pRow->pHead->version); + pRow->pHead.version); - pRow->code = sdbProcessWrite((qtype == TAOS_QTYPE_RPC) ? pRow : NULL, pRow->pHead, qtype, NULL); + pRow->code = sdbProcessWrite((qtype == TAOS_QTYPE_RPC) ? pRow : NULL, &pRow->pHead, qtype, NULL); if (pRow->code > 0) pRow->code = 0; sdbTrace("vgId:1, msg:%p is processed in sdb queue, code:%x", pRow->pMsg, pRow->code); @@ -1099,7 +1119,7 @@ static void *sdbWorkerFp(void *pWorker) { sdbConfirmForward(1, pRow, pRow->code); } else { if (qtype == TAOS_QTYPE_FWD) { - syncConfirmForward(tsSdbMgmt.sync, pRow->pHead->version, pRow->code); + syncConfirmForward(tsSdbMgmt.sync, pRow->pHead.version, pRow->code); } sdbFreeFromQueue(pRow); } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 2da46d5b4bd01d47794b0e77011101ce7dc137cc..4ff2a38dffcb2186f73e2254efeaa6419c68d50b 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -218,7 +218,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) { } pRsp->numOfRows = htonl(rowsRead); - pRsp->precision = htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision + pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.len = size; @@ -280,9 +280,12 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { } } - pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); - pRsp->totalDnodes = htonl(mnodeGetDnodesNum()); - mnodeGetMnodeEpSetForShell(&pRsp->epSet); + int32_t onlineDnodes = 0, totalDnodes = 0; + mnodeGetOnlineAndTotalDnodesNum(&onlineDnodes, &totalDnodes); + + pRsp->onlineDnodes = htonl(onlineDnodes); + pRsp->totalDnodes = htonl(totalDnodes); + mnodeGetMnodeEpSetForShell(&pRsp->epSet, false); pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.len = sizeof(SHeartBeatRsp); @@ -349,7 +352,9 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet); + mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet, false); + + dnodeGetClusterId(pConnectRsp->clusterId); connect_over: if (code != TSDB_CODE_SUCCESS) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2149cb12c0471e747a340a09d869ff6784fc9ccf..1378847d0b472bb65621fc88951d4fbd6e7d0790 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -26,6 +26,7 @@ #include "tcompare.h" #include "tdataformat.h" #include "tgrant.h" +#include "tqueue.h" #include "hash.h" #include "mnode.h" #include "dnode.h" @@ -34,12 +35,9 @@ #include "mnodeAcct.h" #include "mnodeDb.h" #include "mnodeDnode.h" -#include "mnodeMnode.h" -#include "mnodeProfile.h" #include "mnodeSdb.h" #include "mnodeShow.h" #include "mnodeTable.h" -#include "mnodeUser.h" #include "mnodeVgroup.h" #include "mnodeWrite.h" #include "mnodeRead.h" @@ -49,12 +47,14 @@ #define CREATE_CTABLE_RETRY_TIMES 10 #define CREATE_CTABLE_RETRY_SEC 14 -int64_t tsCTableRid = -1; -static void * tsChildTableSdb; -int64_t tsSTableRid = -1; -static void * tsSuperTableSdb; -static int32_t tsChildTableUpdateSize; -static int32_t tsSuperTableUpdateSize; +int64_t tsCTableRid = -1; +static void * tsChildTableSdb; +int64_t tsSTableRid = -1; +static void * tsSuperTableSdb; +static SHashObj *tsSTableUidHash; +static int32_t tsChildTableUpdateSize; +static int32_t tsSuperTableUpdateSize; + static void * mnodeGetChildTable(char *tableId); static void * mnodeGetSuperTable(char *tableId); static void * mnodeGetSuperTableByUid(uint64_t uid); @@ -68,7 +68,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, static int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); - + static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg); @@ -164,7 +164,7 @@ static int32_t mnodeChildTableActionDelete(SSdbRow *pRow) { SVgObj *pVgroup = NULL; SDbObj *pDb = NULL; SAcctObj *pAcct = NULL; - + pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup != NULL) pDb = mnodeGetDb(pVgroup->dbName); if (pDb != NULL) pAcct = mnodeGetAcct(pDb->acct); @@ -180,14 +180,14 @@ static int32_t mnodeChildTableActionDelete(SSdbRow *pRow) { grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1); if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1); } - + if (pDb != NULL) mnodeRemoveTableFromDb(pDb); if (pVgroup != NULL) mnodeRemoveTableFromVgroup(pVgroup, pTable); mnodeDecVgroupRef(pVgroup); mnodeDecDbRef(pDb); mnodeDecAcctRef(pAcct); - + return TSDB_CODE_SUCCESS; } @@ -195,19 +195,19 @@ static int32_t mnodeChildTableActionUpdate(SSdbRow *pRow) { SCTableObj *pNew = pRow->pObj; SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { - void *oldTableId = pTable->info.tableId; + void *oldTableId = pTable->info.tableId; void *oldSql = pTable->sql; void *oldSchema = pTable->schema; void *oldSTable = pTable->superTable; int32_t oldRefCount = pTable->refCount; - + memcpy(pTable, pNew, sizeof(SCTableObj)); - + pTable->refCount = oldRefCount; pTable->sql = pNew->sql; pTable->schema = pNew->schema; pTable->superTable = oldSTable; - + free(pNew); free(oldSql); free(oldSchema); @@ -222,23 +222,23 @@ static int32_t mnodeChildTableActionEncode(SSdbRow *pRow) { SCTableObj *pTable = pRow->pObj; assert(pTable != NULL && pRow->rowData != NULL); - int32_t len = strlen(pTable->info.tableId); + int32_t len = (int32_t)strlen(pTable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pTable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pTable + sizeof(char *), tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - memcpy(pRow->rowData + len, pTable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pTable->schema, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { - memcpy(pRow->rowData + len, pTable->sql, pTable->sqlLen); + memcpy((char *)pRow->rowData + len, pTable->sql, pTable->sqlLen); len += pTable->sqlLen; } } @@ -253,7 +253,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -261,7 +261,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { pTable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pTable + sizeof(char *), pRow->rowData + len, tsChildTableUpdateSize); + memcpy((char *)pTable + sizeof(char *), (char *)pRow->rowData + len, tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { @@ -271,7 +271,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pRow->rowData + len, schemaSize); + memcpy(pTable->schema, (char *)pRow->rowData + len, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { @@ -280,7 +280,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_OUT_OF_MEMORY; } - memcpy(pTable->sql, pRow->rowData + len, pTable->sqlLen); + memcpy(pTable->sql, (char *)pRow->rowData + len, pTable->sqlLen); } } @@ -289,6 +289,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { } static int32_t mnodeChildTableActionRestored() { +#if 0 void *pIter = NULL; SCTableObj *pTable = NULL; @@ -296,7 +297,7 @@ static int32_t mnodeChildTableActionRestored() { pIter = mnodeGetNextChildTable(pIter, &pTable); if (pTable == NULL) break; - SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId); + SDbObj *pDb = mnodeGetDbByTableName(pTable->info.tableId); if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) { mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId); SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; @@ -345,19 +346,20 @@ static int32_t mnodeChildTableActionRestored() { } mnodeCancelGetNextChildTable(pIter); +#endif return 0; } static int32_t mnodeInitChildTables() { SCTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsChildTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_CTABLE, .name = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeChildTableActionInsert, .fpDelete = mnodeChildTableActionDelete, @@ -396,14 +398,15 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { - pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + mDebug("table:%s, create hash:%p", pStable->info.tableId, pStable->vgHash); } if (pStable->vgHash != NULL) { if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) { taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); - mDebug("table:%s, vgId:%d is put into stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, - (int32_t)taosHashGetSize(pStable->vgHash)); + mDebug("table:%s, vgId:%d is put into stable hash:%p, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, + pStable->vgHash, taosHashGetSize(pStable->vgHash)); } } } @@ -416,13 +419,14 @@ static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId); if (pVgroup == NULL) { taosHashRemove(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)); - mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, - (int32_t)taosHashGetSize(pStable->vgHash)); + mDebug("table:%s, vgId:%d is remove from stable hash:%p sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, + pStable->vgHash, taosHashGetSize(pStable->vgHash)); } mnodeDecVgroupRef(pVgroup); } static void mnodeDestroySuperTable(SSTableObj *pStable) { + mDebug("table:%s, is destroyed, stable hash:%p", pStable->info.tableId, pStable->vgHash); if (pStable->vgHash != NULL) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; @@ -439,24 +443,26 @@ static int32_t mnodeSuperTableActionDestroy(SSdbRow *pRow) { static int32_t mnodeSuperTableActionInsert(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; - SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); + SDbObj *pDb = mnodeGetDbByTableName(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { mnodeAddSuperTableIntoDb(pDb); } mnodeDecDbRef(pDb); + taosHashPut(tsSTableUidHash, &pStable->uid, sizeof(int64_t), &pStable, sizeof(int64_t)); return TSDB_CODE_SUCCESS; } static int32_t mnodeSuperTableActionDelete(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; - SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); + SDbObj *pDb = mnodeGetDbByTableName(pStable->info.tableId); if (pDb != NULL) { mnodeRemoveSuperTableFromDb(pDb); mnodeDropAllChildTablesInStable((SSTableObj *)pStable); } mnodeDecDbRef(pDb); + taosHashRemove(tsSTableUidHash, &pStable->uid, sizeof(int64_t)); return TSDB_CODE_SUCCESS; } @@ -464,6 +470,9 @@ static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) { SSTableObj *pNew = pRow->pObj; SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { + mDebug("table:%s, will be updated, hash:%p sizeOfVgList:%d, new hash:%p sizeOfVgList:%d", pTable->info.tableId, + pTable->vgHash, taosHashGetSize(pTable->vgHash), pNew->vgHash, taosHashGetSize(pNew->vgHash)); + void *oldTableId = pTable->info.tableId; void *oldSchema = pTable->schema; void *oldVgHash = pTable->vgHash; @@ -479,6 +488,9 @@ static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) { free(pNew); free(oldTableId); free(oldSchema); + + mDebug("table:%s, update finished, hash:%p sizeOfVgList:%d", pTable->info.tableId, pTable->vgHash, + taosHashGetSize(pTable->vgHash)); } mnodeDecTableRef(pTable); @@ -489,18 +501,18 @@ static int32_t mnodeSuperTableActionEncode(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; assert(pRow->pObj != NULL && pRow->rowData != NULL); - int32_t len = strlen(pStable->info.tableId); + int32_t len = (int32_t)strlen(pStable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pStable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pStable + sizeof(char *), tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - memcpy(pRow->rowData + len, pStable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pStable->schema, schemaSize); len += schemaSize; pRow->rowSize = len; @@ -513,7 +525,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -521,7 +533,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { pStable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pStable + sizeof(char *), pRow->rowData + len, tsSuperTableUpdateSize); + memcpy((char *)pStable + sizeof(char *), (char *)pRow->rowData + len, tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); @@ -531,8 +543,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { return TSDB_CODE_MND_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pRow->rowData + len, schemaSize); - + memcpy(pStable->schema, (char *)pRow->rowData + len, schemaSize); pRow->pObj = pStable; return TSDB_CODE_SUCCESS; @@ -544,14 +555,14 @@ static int32_t mnodeSuperTableActionRestored() { static int32_t mnodeInitSuperTables() { SSTableObj tObj; - tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsSuperTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_STABLE, .name = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeSuperTableActionInsert, .fpDelete = mnodeSuperTableActionDelete, @@ -562,6 +573,7 @@ static int32_t mnodeInitSuperTables() { .fpRestored = mnodeSuperTableActionRestored }; + tsSTableUidHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); tsSTableRid = sdbOpenTable(&desc); tsSuperTableSdb = sdbGetTableByRid(tsSTableRid); if (tsSuperTableSdb == NULL) { @@ -576,6 +588,9 @@ static int32_t mnodeInitSuperTables() { static void mnodeCleanupSuperTables() { sdbCloseTable(tsSTableRid); tsSuperTableSdb = NULL; + + taosHashCleanup(tsSTableUidHash); + tsSTableUidHash = NULL; } int32_t mnodeInitTables() { @@ -595,7 +610,7 @@ int32_t mnodeInitTables() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mnodeProcessAlterTableMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_TABLE_META, mnodeProcessTableMetaMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_STABLE_VGROUP, mnodeProcessSuperTableVgroupMsg); - + mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mnodeProcessCreateChildTableRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mnodeProcessDropChildTableRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mnodeProcessDropSuperTableRsp); @@ -625,20 +640,12 @@ static void *mnodeGetSuperTable(char *tableId) { } static void *mnodeGetSuperTableByUid(uint64_t uid) { - SSTableObj *pStable = NULL; - void *pIter = NULL; + SSTableObj **ppStable = taosHashGet(tsSTableUidHash, &uid, sizeof(int64_t)); + if (ppStable == NULL || *ppStable == NULL) return NULL; - while (1) { - pIter = mnodeGetNextSuperTable(pIter, &pStable); - if (pStable == NULL) break; - if (pStable->uid == uid) { - mnodeCancelGetNextSuperTable(pIter); - return pStable; - } - mnodeDecTableRef(pStable); - } - - return NULL; + SSTableObj *pStable = *ppStable; + mnodeIncTableRef(pStable); + return pStable; } void *mnodeGetTable(char *tableId) { @@ -710,52 +717,198 @@ static void mnodeExtractTableName(char* tableId, char* name) { } } +static SMnodeMsg *mnodeCreateSubMsg(SMnodeMsg *pBatchMasterMsg, int32_t contSize) { + SMnodeMsg *pSubMsg = taosAllocateQitem(sizeof(*pBatchMasterMsg) + contSize); + *pSubMsg = *pBatchMasterMsg; + + //pSubMsg->pCont = (char *) pSubMsg + sizeof(SMnodeMsg); + pSubMsg->rpcMsg.pCont = pSubMsg->pCont; + pSubMsg->successed = 0; + pSubMsg->expected = 0; + SCMCreateTableMsg *pCM = pSubMsg->rpcMsg.pCont; + pCM->numOfTables = htonl(1); + pCM->contLen = htonl(contSize); + + return pSubMsg; +} + +void mnodeDestroySubMsg(SMnodeMsg *pSubMsg) { + if (pSubMsg) { + // pUser is retained in batch master msg + if (pSubMsg->pDb) mnodeDecDbRef(pSubMsg->pDb); + if (pSubMsg->pVgroup) mnodeDecVgroupRef(pSubMsg->pVgroup); + if (pSubMsg->pTable) mnodeDecTableRef(pSubMsg->pTable); + if (pSubMsg->pSTable) mnodeDecTableRef(pSubMsg->pSTable); + if (pSubMsg->pAcct) mnodeDecAcctRef(pSubMsg->pAcct); + if (pSubMsg->pDnode) mnodeDecDnodeRef(pSubMsg->pDnode); + + taosFreeQitem(pSubMsg); + } +} + +static int32_t mnodeValidateCreateTableMsg(SCreateTableMsg *pCreateTable, SMnodeMsg *pMsg) { + if (pMsg->pDb == NULL) { + pMsg->pDb = mnodeGetDbByTableName(pCreateTable->tableName); + } + + if (pMsg->pDb == NULL) { + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableName); + return TSDB_CODE_MND_DB_NOT_SELECTED; + } + + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { + mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); + return TSDB_CODE_MND_DB_IN_DROPPING; + } + + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreateTable->tableName); + if (pMsg->pTable != NULL && pMsg->retry == 0) { + if (pCreateTable->getMeta) { + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableName); + return mnodeGetChildTableMeta(pMsg); + } else if (pCreateTable->igExists) { + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableName); + return TSDB_CODE_SUCCESS; + } else { + mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableName); + return TSDB_CODE_MND_TABLE_ALREADY_EXIST; + } + } + + if (pCreateTable->numOfTags != 0) { + mDebug("msg:%p, app:%p table:%s, batch create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableName, pMsg->rpcMsg.handle); + return mnodeProcessCreateSuperTableMsg(pMsg); + } else { + mDebug("msg:%p, app:%p table:%s, batch create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableName, pMsg->rpcMsg.handle); + return mnodeProcessCreateChildTableMsg(pMsg); + } +} + +static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) { + if (pMsg->pBatchMasterMsg == NULL) { // batch master first round + pMsg->pBatchMasterMsg = pMsg; + + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + int32_t numOfTables = htonl(pCreate->numOfTables); + int32_t contentLen = htonl(pCreate->contLen); + pMsg->expected = numOfTables; + + int32_t code = TSDB_CODE_SUCCESS; + SCreateTableMsg *pCreateTable = (SCreateTableMsg*) ((char*) pCreate + sizeof(SCMCreateTableMsg)); + for (SCreateTableMsg *p = pCreateTable; p < (SCreateTableMsg *) ((char *) pCreate + contentLen); p = (SCreateTableMsg *) ((char *) p + htonl(p->len))) { + SMnodeMsg *pSubMsg = mnodeCreateSubMsg(pMsg, sizeof(SCMCreateTableMsg) + htonl(p->len)); + memcpy(pSubMsg->pCont + sizeof(SCMCreateTableMsg), p, htonl(p->len)); + code = mnodeValidateCreateTableMsg(p, pSubMsg); + + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + ++pSubMsg->pBatchMasterMsg->successed; + mnodeDestroySubMsg(pSubMsg); + continue; + } + + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mnodeDestroySubMsg(pSubMsg); + return code; + } + } + + if (pMsg->successed >= pMsg->expected) { + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + } else { + if (pMsg->pBatchMasterMsg != pMsg) { // batch sub replay + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateTableMsg *pCreateTable = (SCreateTableMsg*) ((char*) pCreate + sizeof(SCMCreateTableMsg)); + int32_t code = mnodeValidateCreateTableMsg(pCreateTable, pMsg); + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + ++pMsg->pBatchMasterMsg->successed; + mnodeDestroySubMsg(pMsg); + } else if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { + return code; + } else if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; + mnodeDestroySubMsg(pMsg); + } + + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->pBatchMasterMsg->code); + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } else { // batch master replay, reprocess the whole batch + assert(0); + return TSDB_CODE_MND_MSG_NOT_PROCESSED; + } + } +} + static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; - - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db); + + int32_t numOfTables = htonl(pCreate->numOfTables); + int32_t contentLen = htonl(pCreate->contLen); + if (numOfTables == 0 || contentLen == 0) { + // todo return error + } + + // batch master msg first round or reprocessing and batch sub msg reprocessing + if (numOfTables > 1 || pMsg->pBatchMasterMsg != NULL) { + return mnodeProcessBatchCreateTableMsg(pMsg); + } + + SCreateTableMsg *p = (SCreateTableMsg*)((char*) pCreate + sizeof(SCMCreateTableMsg)); if (pMsg->pDb == NULL) { - mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + pMsg->pDb = mnodeGetDbByTableName(p->tableName); + } + + if (pMsg->pDb == NULL) { + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return TSDB_CODE_MND_DB_NOT_SELECTED; } - + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); return TSDB_CODE_MND_DB_IN_DROPPING; } - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(p->tableName); if (pMsg->pTable != NULL && pMsg->retry == 0) { - if (pCreate->getMeta) { - mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + if (p->getMeta) { + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return mnodeGetChildTableMeta(pMsg); - } else if (pCreate->igExists) { - mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + } else if (p->igExists) { + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return TSDB_CODE_SUCCESS; } else { - mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return TSDB_CODE_MND_TABLE_ALREADY_EXIST; } } - if (pCreate->numOfTags != 0) { + if (p->numOfTags != 0) { mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, pMsg->rpcMsg.handle); + p->tableName, pMsg->rpcMsg.handle); return mnodeProcessCreateSuperTableMsg(pMsg); } else { mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, pMsg->rpcMsg.handle); + p->tableName, pMsg->rpcMsg.handle); return mnodeProcessCreateChildTableMsg(pMsg); } } static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont; - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(pDrop->name); if (pMsg->pDb == NULL) { mError("msg:%p, app:%p table:%s, failed to drop table, db not selected or db in dropping", pMsg, - pMsg->rpcMsg.ahandle, pDrop->tableId); + pMsg->rpcMsg.ahandle, pDrop->name); return TSDB_CODE_MND_DB_NOT_SELECTED; } @@ -764,32 +917,34 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_DB_IN_DROPPING; } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("msg:%p, app:%p table:%s, failed to drop table, in monitor database", pMsg, pMsg->rpcMsg.ahandle, - pDrop->tableId); + pDrop->name); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->name); if (pMsg->pTable == NULL) { if (pDrop->igNotExists) { - mDebug("msg:%p, app:%p table:%s is not exist, treat as success", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId); + mDebug("msg:%p, app:%p table:%s is not exist, treat as success", pMsg, pMsg->rpcMsg.ahandle, pDrop->name); return TSDB_CODE_SUCCESS; } else { - mError("msg:%p, app:%p table:%s, failed to drop, table not exist", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId); + mError("msg:%p, app:%p table:%s, failed to drop, table not exist", pMsg, pMsg->rpcMsg.ahandle, pDrop->name); return TSDB_CODE_MND_INVALID_TABLE_NAME; } } if (pMsg->pTable->type == TSDB_SUPER_TABLE) { SSTableObj *pSTable = (SSTableObj *)pMsg->pTable; - mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", - pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash)); + mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", pMsg, + pMsg->rpcMsg.ahandle, pDrop->name, pSTable->uid, pSTable->numOfTables, taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { SCTableObj *pCTable = (SCTableObj *)pMsg->pTable; mInfo("msg:%p, app:%p table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, - pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid); + pDrop->name, pCTable->vgId, pCTable->tid, pCTable->uid); return mnodeProcessDropChildTableMsg(pMsg); } } @@ -798,29 +953,29 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->createFlag = htons(pInfo->createFlag); mDebug("msg:%p, app:%p table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); + pInfo->tableFname, pMsg->rpcMsg.handle, pInfo->createFlag); - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(pInfo->tableFname); if (pMsg->pDb == NULL) { mError("msg:%p, app:%p table:%s, failed to get table meta, db not selected", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId); + pInfo->tableFname); return TSDB_CODE_MND_DB_NOT_SELECTED; } - + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); return TSDB_CODE_MND_DB_IN_DROPPING; } - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableFname); if (pMsg->pTable == NULL) { if (!pInfo->createFlag) { mError("msg:%p, app:%p table:%s, failed to get table meta, table not exist", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId); + pInfo->tableFname); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { mDebug("msg:%p, app:%p table:%s, failed to get table meta, start auto create table ", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId); + pInfo->tableFname); return mnodeAutoCreateChildTable(pMsg); } } else { @@ -851,15 +1006,21 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + SCMCreateTableMsg *pCreate1 = pMsg->rpcMsg.pCont; + if (pCreate1->numOfTables == 0) { + return TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG; + } + + SCreateTableMsg* pCreate = (SCreateTableMsg*)((char*)pCreate1 + sizeof(SCMCreateTableMsg)); + SSTableObj * pStable = calloc(1, sizeof(SSTableObj)); if (pStable == NULL) { - mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_OUT_OF_MEMORY; } int64_t us = taosGetTimestampUs(); - pStable->info.tableId = strdup(pCreate->tableId); + pStable->info.tableId = strdup(pCreate->tableName); pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); pStable->uid = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); @@ -873,19 +1034,28 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pStable->schema = (SSchema *)calloc(1, schemaSize); if (pStable->schema == NULL) { free(pStable); - mError("msg:%p, app:%p table:%s, failed to create, no schema input", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, no schema input", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_INVALID_TABLE_NAME; } + memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); + if (pStable->numOfColumns > TSDB_MAX_COLUMNS || pStable->numOfTags > TSDB_MAX_TAGS) { + mError("msg:%p, app:%p table:%s, failed to create, too many columns", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); + return TSDB_CODE_MND_INVALID_TABLE_NAME; + } + pStable->nextColId = 0; + for (int32_t col = 0; col < numOfCols; col++) { SSchema *tschema = pStable->schema; tschema[col].colId = pStable->nextColId++; tschema[col].bytes = htons(tschema[col].bytes); - - // todo 1. check the length of each column; 2. check the total length of all columns - assert(tschema[col].type >= TSDB_DATA_TYPE_BOOL && tschema[col].type <= TSDB_DATA_TYPE_NCHAR); + } + + if (!tIsValidSchema(pStable->schema, pStable->numOfColumns, pStable->numOfTags)) { + mError("msg:%p, app:%p table:%s, failed to create table, invalid schema", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); + return TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG; } pMsg->pTable = (STableObj *)pStable; @@ -904,7 +1074,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroySuperTable(pStable); pMsg->pTable = NULL; - mError("msg:%p, app:%p table:%s, failed to create, sdb error", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, sdb error", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); } return code; @@ -914,18 +1084,14 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pTable = (SSTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); - } else { - mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); + + return code; } - return code; -} - -static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { - if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; + mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { + if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL); while (pVgId) { SVgObj *pVgroup = mnodeGetVgroup(*pVgId); @@ -936,10 +1102,11 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { pDrop->contLen = htonl(sizeof(SDropSTableMsg)); pDrop->vgId = htonl(pVgroup->vgId); pDrop->uid = htobe64(pStable->uid); - mnodeExtractTableName(pStable->info.tableId, pDrop->tableId); + mnodeExtractTableName(pStable->info.tableId, pDrop->tableFname); - mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d", pMsg, pMsg->rpcMsg.ahandle, - pStable->info.tableId, pVgroup->vgId); + mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d, hash:%p sizeOfVgList:%d", pMsg, + pMsg->rpcMsg.ahandle, pStable->info.tableId, pVgroup->vgId, pStable->vgHash, + taosHashGetSize(pStable->vgHash)); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup); SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; dnodeSendMsgToDnode(&epSet, &rpcMsg); @@ -949,8 +1116,18 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { taosHashCancelIterate(pStable->vgHash, pVgId); mnodeDropAllChildTablesInStable(pStable); - } - + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { + if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; + + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; + mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash)); + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, @@ -1086,7 +1263,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c } // int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + uint32_t len = (int32_t)strlen(newTagName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1094,7 +1271,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c if (mnodeFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_MND_TAG_ALREAY_EXIST; } - + // update SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); tstrncpy(schema->name, newTagName, sizeof(schema->name)); @@ -1249,7 +1426,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1257,7 +1434,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char if (mnodeFindSuperTableColumnIndex(pStable, newName) >= 0) { return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } - + // update SSchema *schema = (SSchema *) (pStable->schema + col); tstrncpy(schema->name, newName, sizeof(schema->name)); @@ -1280,7 +1457,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -1290,9 +1467,9 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SSchema tbnameSchema = tGetTableNameColumnSchema(); - pShow->bytes[cols] = tbnameSchema.bytes; - pSchema[cols].type = tbnameSchema.type; + SSchema* tbnameSchema = tGetTbnameColumnSchema(); + pShow->bytes[cols] = tbnameSchema->bytes; + pSchema[cols].type = tbnameSchema->type; strcpy(pSchema[cols].name, "name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -1345,7 +1522,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -1354,12 +1531,12 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - prefixLen = strlen(prefix); + prefixLen = (int32_t)strlen(prefix); SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char stableName[TSDB_TABLE_NAME_LEN] = {0}; - while (numOfRows < rows) { + while (numOfRows < rows) { pShow->pIter = mnodeGetNextSuperTable(pShow->pIter, &pTable); if (pTable == NULL) break; if (strncmp(pTable->info.tableId, prefix, prefixLen)) { @@ -1378,11 +1555,11 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - - int16_t len = strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); + + int16_t len = (int16_t)strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); *(int16_t*) pWrite = len; pWrite += sizeof(int16_t); // todo refactor - + strncpy(pWrite, stableName, len); cols++; @@ -1422,7 +1599,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all super tables will be dropped from sdb", pDropDb->name); @@ -1449,7 +1626,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; assert(numOfCols <= TSDB_MAX_COLUMNS); - + for (int32_t i = 0; i < numOfCols; ++i) { tstrncpy(pSchema->name, pTable->schema[i].name, sizeof(pSchema->name)); pSchema->type = pTable->schema[i].type; @@ -1475,15 +1652,15 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromSuperTable(pMeta->schema, pTable); - tstrncpy(pMeta->tableId, pTable->info.tableId, sizeof(pMeta->tableId)); + tstrncpy(pMeta->tableFname, pTable->info.tableId, sizeof(pMeta->tableFname)); pMsg->rpcRsp.len = pMeta->contLen; pMeta->contLen = htons(pMeta->contLen); pMsg->rpcRsp.rsp = pMeta; - mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg, pMsg->rpcMsg.ahandle, - pTable->info.tableId, pTable->uid); + mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved, sizeOfVgList:%d numOfTables:%d", pMsg, + pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->uid, taosHashGetSize(pTable->vgHash), pTable->numOfTables); return TSDB_CODE_SUCCESS; } @@ -1512,7 +1689,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; + char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("msg:%p, app:%p stable:%s, not exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, stableName); @@ -1529,10 +1706,12 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; pVgroupMsg->numOfVgroups = 0; - + msg += sizeof(SVgroupsMsg); } else { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; + mDebug("msg:%p, app:%p stable:%s, hash:%p sizeOfVgList:%d will be returned", pMsg, pMsg->rpcMsg.ahandle, + pTable->info.tableId, pTable->vgHash, taosHashGetSize(pTable->vgHash)); int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL); int32_t vgSize = 0; @@ -1573,9 +1752,9 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { rpcFreeCont(pRsp); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { - pRsp->numOfTables = htonl(pRsp->numOfTables); + pRsp->numOfTables = (int32_t)htonl(pRsp->numOfTables); pMsg->rpcRsp.rsp = pRsp; - pMsg->rpcRsp.len = msg - (char *)pRsp; + pMsg->rpcRsp.len = (int32_t)((char *)msg - (char *)pRsp); return TSDB_CODE_SUCCESS; } @@ -1585,8 +1764,11 @@ static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg) { mInfo("drop stable rsp received, result:%s", tstrerror(rpcMsg->code)); } -static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj *pTable) { - STagData * pTagData = NULL; +static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pCreateMsg, SCTableObj *pTable) { + SCreateTableMsg* pMsg = (SCreateTableMsg*) ((char*)pCreateMsg + sizeof(SCMCreateTableMsg)); + + char* tagData = NULL; + int32_t tagDataLen = 0; int32_t totalCols = 0; int32_t contLen = 0; @@ -1594,9 +1776,13 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj * totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen; if (pMsg != NULL) { - pTagData = (STagData *)pMsg->schema; - tagDataLen = htonl(pTagData->dataLen); + int32_t nameLen = htonl(*(int32_t*)pMsg->schema); + char* p = pMsg->schema + nameLen + sizeof(int32_t); + + tagDataLen = htonl(*(int32_t*) p); contLen += tagDataLen; + + tagData = p + sizeof(int32_t); } } else { totalCols = pTable->numOfColumns; @@ -1609,7 +1795,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj * return NULL; } - mnodeExtractTableName(pTable->info.tableId, pCreate->tableId); + mnodeExtractTableName(pTable->info.tableId, pCreate->tableFname); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); pCreate->tableType = pTable->info.type; @@ -1617,9 +1803,9 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj * pCreate->tid = htonl(pTable->tid); pCreate->sqlDataLen = htonl(pTable->sqlLen); pCreate->uid = htobe64(pTable->uid); - + if (pTable->info.type == TSDB_CHILD_TABLE) { - mnodeExtractTableName(pTable->superTable->info.tableId, pCreate->superTableId); + mnodeExtractTableName(pTable->superTable->info.tableId, pCreate->stableFname); pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); pCreate->numOfTags = htons(pTable->superTable->numOfTags); pCreate->sversion = htonl(pTable->superTable->sversion); @@ -1634,7 +1820,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj * pCreate->tagDataLen = 0; pCreate->superTableUid = 0; } - + SSchema *pSchema = (SSchema *) pCreate->data; if (pTable->info.type == TSDB_CHILD_TABLE) { memcpy(pSchema, pTable->superTable->schema, totalCols * sizeof(SSchema)); @@ -1648,7 +1834,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj * } if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) { - memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData->data, tagDataLen); + memcpy(pCreate->data + totalCols * sizeof(SSchema), tagData, tagDataLen); } if (pTable->info.type == TSDB_STREAM_TABLE) { @@ -1686,7 +1872,8 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + + SCreateTableMsg *pCreate = (SCreateTableMsg*) ((char*)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); assert(pTable); if (code == TSDB_CODE_SUCCESS) { @@ -1700,6 +1887,18 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { mDebug("msg:%p, app:%p table:%s, created in dnode, thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->rpcMsg.handle); + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->successed; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -1708,36 +1907,62 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + return code; } } static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SVgObj *pVgroup = pMsg->pVgroup; - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + + SCMCreateTableMsg *p1 = pMsg->rpcMsg.pCont; + SCreateTableMsg *pCreate = (SCreateTableMsg*)((char*)p1 + sizeof(SCMCreateTableMsg)); + SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) { - mError("msg:%p, app:%p table:%s, failed to alloc memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to alloc memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_OUT_OF_MEMORY; } - if (pCreate->numOfColumns == 0) { - pTable->info.type = TSDB_CHILD_TABLE; - } else { - pTable->info.type = TSDB_NORMAL_TABLE; - } - - pTable->info.tableId = strdup(pCreate->tableId); + pTable->info.type = (pCreate->numOfColumns == 0)? TSDB_CHILD_TABLE:TSDB_NORMAL_TABLE; + pTable->info.tableId = strdup(pCreate->tableName); pTable->createdTime = taosGetTimestampMs(); pTable->tid = tid; pTable->vgId = pVgroup->vgId; if (pTable->info.type == TSDB_CHILD_TABLE) { - STagData *pTagData = (STagData *)pCreate->schema; // it is a tag key - if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name); + int32_t nameLen = htonl(*(int32_t*) pCreate->schema); + char* name = (char*)pCreate->schema + sizeof(int32_t); + + char stableName[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(stableName, name, nameLen); + + char prefix[64] = {0}; + size_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64); + if (0 != strncasecmp(prefix, stableName, prefixLen)) { + mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle, + pCreate->tableName, stableName); + mnodeDestroyChildTable(pTable); + return TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; + } + + if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(stableName); if (pMsg->pSTable == NULL) { mError("msg:%p, app:%p table:%s, corresponding super table:%s does not exist", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, pTagData->name); + pCreate->tableName, stableName); mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -1799,12 +2024,12 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { .pMsg = pMsg, .fpReq = mnodeDoCreateChildTableFp }; - + int32_t code = sdbInsertRow(&desc); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroyChildTable(pTable); pMsg->pTable = NULL; - mError("msg:%p, app:%p table:%s, failed to create, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, + mError("msg:%p, app:%p table:%s, failed to create, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName, tstrerror(code)); } else { mDebug("msg:%p, app:%p table:%s, allocated in vgroup, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, @@ -1815,11 +2040,13 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + //SCMCreateTableMsg* p1 = pMsg->rpcMsg.pCont; // there are several tables here. + SCreateTableMsg* pCreate = (SCreateTableMsg*)((char *)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); + int32_t code = grantCheck(TSDB_GRANT_TIMESERIES); if (code != TSDB_CODE_SUCCESS) { mError("msg:%p, app:%p table:%s, failed to create, grant timeseries failed", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId); + pCreate->tableName); return code; } @@ -1830,7 +2057,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &tid); if (code != TSDB_CODE_SUCCESS) { mDebug("msg:%p, app:%p table:%s, failed to get available vgroup, reason:%s", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, tstrerror(code)); + pCreate->tableName, tstrerror(code)); return code; } @@ -1844,15 +2071,15 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { return mnodeDoCreateChildTable(pMsg, tid); } } else { - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableName); } if (pMsg->pTable == NULL) { - mError("msg:%p, app:%p table:%s, object not found, retry:%d reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pMsg->retry, + mError("msg:%p, app:%p table:%s, object not found, retry:%d reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName, pMsg->retry, tstrerror(terrno)); return terrno; } else { - mDebug("msg:%p, app:%p table:%s, send create msg to vnode again", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mDebug("msg:%p, app:%p table:%s, send create msg to vnode again", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return mnodeDoCreateChildTableFp(pMsg); } } @@ -1868,7 +2095,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); + tstrncpy(pDrop->tableFname, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); pDrop->vgId = htonl(pTable->vgId); pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); pDrop->tid = htonl(pTable->tid); @@ -1877,7 +2104,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); mInfo("msg:%p, app:%p ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, - pDrop->tableId, pTable->vgId, pTable->tid, pTable->uid); + pDrop->tableFname, pTable->vgId, pTable->tid, pTable->uid); SRpcMsg rpcMsg = { .ahandle = pMsg, @@ -1899,7 +2126,7 @@ static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; mError("msg:%p, app:%p ctable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); return code; - } + } return mnodeSendDropChildTableMsg(pMsg, true); } @@ -2008,7 +2235,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 pTable->numOfColumns += ncols; pTable->sversion++; - + SAcctObj *pAcct = mnodeGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += ncols; @@ -2071,7 +2298,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -2079,7 +2306,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char if (mnodeFindNormalTableColumnIndex(pTable, newName) >= 0) { return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } - + // update SSchema *schema = (SSchema *) (pTable->schema + col); tstrncpy(schema->name, newName, sizeof(schema->name)); @@ -2119,12 +2346,13 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->tid = htonl(pTable->tid); pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; - tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); - if (pTable->superTable != NULL) { - tstrncpy(pMeta->sTableId, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); - } + tstrncpy(pMeta->tableFname, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); + + if (pTable->info.type == TSDB_CHILD_TABLE) { + assert(pTable->superTable != NULL); + tstrncpy(pMeta->sTableName, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); - if (pTable->info.type == TSDB_CHILD_TABLE && pTable->superTable != NULL) { + pMeta->suid = pTable->superTable->uid; pMeta->sversion = htons(pTable->superTable->sversion); pMeta->tversion = htons(pTable->superTable->tversion); pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags; @@ -2135,7 +2363,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->tversion = 0; pMeta->numOfTags = 0; pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); - pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromNormalTable(pMeta->schema, pTable); + pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromNormalTable(pMeta->schema, pTable); } if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); @@ -2163,32 +2391,53 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; - STagData *pTags = (STagData *)pInfo->tags; - int32_t tagLen = htonl(pTags->dataLen); - if (pTags->name[0] == 0) { - mError("msg:%p, app:%p table:%s, failed to create table on demand for stable is empty, tagLen:%d", pMsg, - pMsg->rpcMsg.ahandle, pInfo->tableId, tagLen); + + if (pMsg->rpcMsg.contLen <= sizeof(*pInfo)) { + mError("msg:%p, app:%p table:%s, failed to auto create child table, tags not exist", pMsg, pMsg->rpcMsg.ahandle, + pInfo->tableFname); + return TSDB_CODE_MND_TAG_NOT_EXIST; + } + + char* p = pInfo->tags; + int32_t nameLen = htonl(*(int32_t*) p); + p += sizeof(int32_t); + p += nameLen; + + int32_t tagLen = htonl(*(int32_t*) p); + p += sizeof(int32_t); + + int32_t totalLen = nameLen + tagLen + sizeof(int32_t)*2; + if (tagLen == 0 || nameLen == 0) { + mError("msg:%p, app:%p table:%s, failed to create table on demand for super table is empty, tagLen:%d", pMsg, + pMsg->rpcMsg.ahandle, pInfo->tableFname, tagLen); return TSDB_CODE_MND_INVALID_STABLE_NAME; } - int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen; + int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg) + totalLen; SCMCreateTableMsg *pCreateMsg = calloc(1, contLen); if (pCreateMsg == NULL) { mError("msg:%p, app:%p table:%s, failed to create table while get meta info, no enough memory", pMsg, - pMsg->rpcMsg.ahandle, pInfo->tableId); + pMsg->rpcMsg.ahandle, pInfo->tableFname); return TSDB_CODE_MND_OUT_OF_MEMORY; } - size_t size = sizeof(pInfo->tableId); - tstrncpy(pCreateMsg->tableId, pInfo->tableId, size); - tstrncpy(pCreateMsg->db, pMsg->pDb->name, sizeof(pCreateMsg->db)); - pCreateMsg->igExists = 1; - pCreateMsg->getMeta = 1; + SCreateTableMsg* pCreate = (SCreateTableMsg*) ((char*) pCreateMsg + sizeof(SCMCreateTableMsg)); + + size_t size = tListLen(pInfo->tableFname); + tstrncpy(pCreate->tableName, pInfo->tableFname, size); + pCreate->igExists = 1; + pCreate->getMeta = 1; + + pCreateMsg->numOfTables = htonl(1); pCreateMsg->contLen = htonl(contLen); - memcpy(pCreateMsg->schema, pTags, contLen - sizeof(SCMCreateTableMsg)); + memcpy(pCreate->schema, pInfo->tags, totalLen); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(name, pInfo->tags + sizeof(int32_t), nameLen); + mDebug("msg:%p, app:%p table:%s, start to create on demand, tagLen:%d stable:%s", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId, tagLen, pTags->name); + pInfo->tableFname, tagLen, name); if (pMsg->rpcMsg.pCont != pMsg->pCont) { tfree(pMsg->rpcMsg.pCont); @@ -2252,7 +2501,7 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all child tables will be dropped from sdb", pDropDb->name); @@ -2318,7 +2567,7 @@ static SCTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { return TSDB_CODE_COM_OPS_NOT_SUPPORT; -#if 0 +#if 0 SConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); @@ -2336,13 +2585,13 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { SMDCreateTableMsg *pCreate = NULL; pCreate = mnodeBuildCreateChildTableMsg(NULL, (SCTableObj *)pTable); mnodeDecTableRef(pTable); - + if (pCreate == NULL) return terrno; - + pMsg->rpcRsp.rsp = pCreate; pMsg->rpcRsp.len = htonl(pCreate->contLen); return TSDB_CODE_SUCCESS; -#endif +#endif } // handle drop child response @@ -2410,6 +2659,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeSendDropChildTableMsg(pMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->successed; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); return; } @@ -2422,11 +2684,25 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { .pMsg = pMsg, .fpRsp = mnodeDoCreateChildTableCb }; - + int32_t code = sdbInsertRowToQueue(&desc); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { pMsg->pTable = NULL; mnodeDestroyChildTable(pTable); + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, code); } } else { @@ -2452,6 +2728,20 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { //Avoid retry again in client rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY; } + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = rpcMsg->code; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); } } @@ -2504,7 +2794,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { SCTableObj *pTable = mnodeGetChildTable(tableId); if (pTable == NULL) continue; - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(tableId); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mnodeDecTableRef(pTable); continue; @@ -2543,7 +2833,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -2553,9 +2843,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SSchema s = tGetTableNameColumnSchema(); - pShow->bytes[cols] = s.bytes; - pSchema[cols].type = s.type; + SSchema* s = tGetTbnameColumnSchema(); + pShow->bytes[cols] = s->bytes; + pSchema[cols].type = s->type; strcpy(pSchema[cols].name, "table_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -2572,9 +2862,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - SSchema tbCol = tGetTableNameColumnSchema(); - pShow->bytes[cols] = tbCol.bytes + VARSTR_HEADER_SIZE; - pSchema[cols].type = tbCol.type; + SSchema* tbCol = tGetTbnameColumnSchema(); + pShow->bytes[cols] = tbCol->bytes + VARSTR_HEADER_SIZE; + pSchema[cols].type = tbCol->type; strcpy(pSchema[cols].name, "stable_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -2616,7 +2906,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -2629,9 +2919,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; - tstrncpy(prefix, pDb->name, 64); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)tableIdPrefix(pDb->name, prefix, 64); char* pattern = NULL; if (pShow->payloadLen > 0) { @@ -2655,7 +2943,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows } char tableName[TSDB_TABLE_NAME_LEN] = {0}; - + // pattern compare for table name mnodeExtractTableName(pTable->info.tableId, tableName); @@ -2684,13 +2972,13 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - + memset(tableName, 0, sizeof(tableName)); if (pTable->info.type == TSDB_CHILD_TABLE) { mnodeExtractTableName(pTable->superTable->info.tableId, tableName); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, pShow->bytes[cols]); } - + cols++; // uid @@ -2725,27 +3013,29 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; mDebug("msg:%p, app:%p table:%s, alter table msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pAlter->tableId, pMsg->rpcMsg.handle); + pAlter->tableFname, pMsg->rpcMsg.handle); - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(pAlter->tableFname); if (pMsg->pDb == NULL) { - mError("msg:%p, app:%p table:%s, failed to alter table, db not selected", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, db not selected", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); return TSDB_CODE_MND_DB_NOT_SELECTED; } - + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); return TSDB_CODE_MND_DB_IN_DROPPING; } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { - mError("msg:%p, app:%p table:%s, failed to alter table, its log db", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, its log db", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableFname); if (pMsg->pTable == NULL) { - mError("msg:%p, app:%p table:%s, failed to alter table, table not exist", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, table not exist", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -2754,7 +3044,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { pAlter->tagValLen = htonl(pAlter->tagValLen); if (pAlter->numOfCols > 2) { - mError("msg:%p, app:%p table:%s, error numOfCols:%d in alter table", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId, + mError("msg:%p, app:%p table:%s, error numOfCols:%d in alter table", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname, pAlter->numOfCols); return TSDB_CODE_MND_APP_ERROR; } @@ -2765,7 +3055,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { int32_t code = TSDB_CODE_COM_OPS_NOT_SUPPORT; if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - mDebug("msg:%p, app:%p table:%s, start to alter stable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mDebug("msg:%p, app:%p table:%s, start to alter stable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { code = mnodeAddSuperTableTag(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { @@ -2781,7 +3071,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { } else { } } else { - mDebug("msg:%p, app:%p table:%s, start to alter ctable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mDebug("msg:%p, app:%p table:%s, start to alter ctable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { return TSDB_CODE_COM_OPS_NOT_SUPPORT; } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { @@ -2800,7 +3090,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -2810,9 +3100,9 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SSchema tbnameColSchema = tGetTableNameColumnSchema(); - pShow->bytes[cols] = tbnameColSchema.bytes; - pSchema[cols].type = tbnameColSchema.type; + SSchema* tbnameColSchema = tGetTbnameColumnSchema(); + pShow->bytes[cols] = tbnameColSchema->bytes; + pSchema[cols].type = tbnameColSchema->type; strcpy(pSchema[cols].name, "table_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -2853,13 +3143,13 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); return 0; } - + int32_t numOfRows = 0; SCTableObj *pTable = NULL; SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; @@ -2867,12 +3157,12 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro char prefix[64] = {0}; tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); while (numOfRows < rows) { pShow->pIter = mnodeGetNextChildTable(pShow->pIter, &pTable); if (pTable == NULL) break; - + // not belong to current db if (strncmp(pTable->info.tableId, prefix, prefixLen) || pTable->info.type != TSDB_STREAM_TABLE) { mnodeDecTableRef(pTable); @@ -2880,7 +3170,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro } char tableName[TSDB_TABLE_NAME_LEN] = {0}; - + // pattern compare for table name mnodeExtractTableName(pTable->info.tableId, tableName); diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index fb26086d0433d7760c9cc7ed906abb570a4fcfee..55ee39b6bceb7a3d5ebc8838e1a9a2151ccb6c3d 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -123,12 +123,12 @@ static void mnodePrintUserAuth() { mnodeDecUserRef(pUser); } - fflush(fp); + fsync(fileno(fp)); fclose(fp); } static int32_t mnodeUserActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsUserSdb); + int64_t numOfRows = sdbGetNumOfRows(tsUserSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root user"); SAcctObj *pAcct = mnodeGetAcct(TSDB_DEFAULT_USER); @@ -148,14 +148,14 @@ static int32_t mnodeUserActionRestored() { int32_t mnodeInitUsers() { SUserObj tObj; - tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsUserUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_USER, .name = "users", .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE, .maxRowSize = tsUserUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeUserActionInsert, .fpDelete = mnodeUserActionDelete, @@ -204,11 +204,11 @@ void mnodeCancelGetNextUser(void *pIter) { } void mnodeIncUserRef(SUserObj *pUser) { - return sdbIncRef(tsUserSdb, pUser); + sdbIncRef(tsUserSdb, pUser); } void mnodeDecUserRef(SUserObj *pUser) { - return sdbDecRef(tsUserSdb, pUser); + sdbDecRef(tsUserSdb, pUser); } static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { @@ -337,7 +337,7 @@ static int32_t mnodeGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo cols++; pMeta->numOfColumns = htons(cols); - strcpy(pMeta->tableId, "show users"); + strcpy(pMeta->tableFname, "show users"); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -561,7 +561,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) { void mnodeDropAllUsers(SAcctObj *pAcct) { void * pIter = NULL; int32_t numOfUsers = 0; - int32_t acctNameLen = strlen(pAcct->user); + int32_t acctNameLen = (int32_t)strlen(pAcct->user); SUserObj *pUser = NULL; while (1) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index d3020de6bd32dfcee97a068181d1e957e7a1a62a..98eab3f1ed07545974bc3247275530c81fd34e84 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -184,6 +184,7 @@ static int32_t mnodeVgroupActionEncode(SSdbRow *pRow) { for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { pTmpVgroup->vnodeGid[i].pDnode = NULL; pTmpVgroup->vnodeGid[i].role = 0; + memset(pTmpVgroup->vnodeGid[i].vver, 0, sizeof(pTmpVgroup->vnodeGid[i].vver)); } pRow->rowSize = tsVgUpdateSize; @@ -205,14 +206,14 @@ static int32_t mnodeVgroupActionRestored() { int32_t mnodeInitVgroups() { SVgObj tObj; - tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsVgUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_VGROUP, .name = "vgroups", .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE, .maxRowSize = tsVgUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeVgroupActionInsert, .fpDelete = mnodeVgroupActionDelete, @@ -244,11 +245,11 @@ int32_t mnodeInitVgroups() { } void mnodeIncVgroupRef(SVgObj *pVgroup) { - return sdbIncRef(tsVgroupSdb, pVgroup); + sdbIncRef(tsVgroupSdb, pVgroup); } void mnodeDecVgroupRef(SVgObj *pVgroup) { - return sdbDecRef(tsVgroupSdb, pVgroup); + sdbDecRef(tsVgroupSdb, pVgroup); } SVgObj *mnodeGetVgroup(int32_t vgId) { @@ -256,6 +257,8 @@ SVgObj *mnodeGetVgroup(int32_t vgId) { } void mnodeUpdateVgroup(SVgObj *pVgroup) { + pVgroup->vgCfgVersion++; + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, @@ -315,8 +318,10 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; if (pVgid->pDnode == pDnode) { - mTrace("dnode:%d, receive status from dnode, vgId:%d status is %d:%s", pDnode->dnodeId, pVgroup->vgId, pVgid->role, syncRole[pVgid->role]); + mTrace("vgId:%d, receive vnode status from dnode:%d, status:%s last:%s vver:%" PRIu64, pVgroup->vgId, + pDnode->dnodeId, syncRole[pVload->role], syncRole[pVgid->role], pVload->vnodeVersion); pVgid->role = pVload->role; + mnodeSetVgidVer(pVgid->vver, pVload->vnodeVersion); if (pVload->role == TAOS_SYNC_ROLE_MASTER) { pVgroup->inUse = i; } @@ -338,10 +343,11 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl pVgroup->pointsWritten = htobe64(pVload->pointsWritten); } - if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) { - mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mnode cfgVersion:%d replica:%d", - pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion, - pVgroup->numOfVnodes); + if (pVload->dbCfgVersion != pVgroup->pDb->dbCfgVersion || pVload->replica != pVgroup->numOfVnodes || + pVload->vgCfgVersion != pVgroup->vgCfgVersion) { + mError("dnode:%d, vgId:%d, vnode cfgVersion:%d:%d repica:%d not match with mnode cfgVersion:%d:%d replica:%d", + pDnode->dnodeId, pVload->vgId, pVload->dbCfgVersion, pVload->vgCfgVersion, pVload->replica, + pVgroup->pDb->dbCfgVersion, pVgroup->vgCfgVersion, pVgroup->numOfVnodes); mnodeSendAlterVgroupMsg(pVgroup); } } @@ -528,6 +534,20 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { tstrerror(code)); SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; sdbDeleteRow(&desc); + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = pMsg->code; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + return code; } else { mInfo("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, @@ -559,7 +579,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { SDbObj *pDb = pMsg->pDb; SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); - tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_LEN + TSDB_DB_NAME_LEN); + tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN); pVgroup->numOfVnodes = pDb->cfg.replications; pVgroup->createdTime = taosGetTimestampMs(); pVgroup->accessState = TSDB_VN_ALL_ACCCESS; @@ -655,7 +675,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "onlineVnodes"); + strcpy(pSchema[cols].name, "onlines"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -670,13 +690,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -839,7 +859,8 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { SVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); - pCfg->cfgVersion = htonl(pDb->cfgVersion); + pCfg->dbCfgVersion = htonl(pDb->dbCfgVersion); + pCfg->vgCfgVersion = htonl(pVgroup->vgCfgVersion); pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); pCfg->maxTables = htonl(maxTables + 1); @@ -854,10 +875,12 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; pCfg->walLevel = pDb->cfg.walLevel; - pCfg->replications = (int8_t) pVgroup->numOfVnodes; + pCfg->vgReplica = (int8_t) pVgroup->numOfVnodes; pCfg->wals = 3; pCfg->quorum = pDb->cfg.quorum; pCfg->update = pDb->cfg.update; + pCfg->cacheLastRow = pDb->cfg.cacheLastRow; + pCfg->dbReplica = pDb->cfg.replications; SVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { @@ -977,6 +1000,20 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); + + if (mnodeMsg->pBatchMasterMsg) { + ++mnodeMsg->pBatchMasterMsg->received; + mnodeMsg->pBatchMasterMsg->code = code; + if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received + >= mnodeMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(mnodeMsg); + + return; + } + dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { @@ -986,6 +1023,20 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { .pObj = pVgroup }; sdbDeleteRow(&row); + + if (mnodeMsg->pBatchMasterMsg) { + ++mnodeMsg->pBatchMasterMsg->received; + mnodeMsg->pBatchMasterMsg->code = mnodeMsg->code; + if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received + >= mnodeMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, mnodeMsg->code); + } + + mnodeDestroySubMsg(mnodeMsg); + + return; + } + dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); } } @@ -1173,3 +1224,14 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) { mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups); } + +int32_t mnodeGetVgidVer(int8_t *cver) { + int32_t iver = ((int32_t)cver[0]) * 10000 + ((int32_t)cver[1]) * 100 + (int32_t)cver[2]; + return iver; +} + +void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { + cver[0] = (int8_t)((int32_t)(iver % 1000000) / 10000); + cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); + cver[2] = (int8_t)(iver % 100); +} diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index 53981238a76ee3976689f77315eabab118816e81..c0699b05b364927492b8c2656bead0e14d46ab5a 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -50,7 +50,7 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); - mnodeGetMnodeEpSetForShell(epSet); + mnodeGetMnodeEpSetForShell(epSet, true); rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index 4e44d29a025eee1eb8b850494b235dc48e9c050a..ab8b0f76785c24e3385f49245f6e191b2d57cc40 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -10,3 +10,5 @@ ELSEIF (TD_WINDOWS) ENDIF () ADD_SUBDIRECTORY(src/detail) +ADD_SUBDIRECTORY(tests) + diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h new file mode 100644 index 0000000000000000000000000000000000000000..0874ca975b44a2e355c7eb746af4f30b4d163610 --- /dev/null +++ b/src/os/inc/eok.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ +#define _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __APPLE__ + +enum EPOLL_EVENTS + { + EPOLLIN = 0x001, +#define EPOLLIN EPOLLIN + EPOLLPRI = 0x002, +#define EPOLLPRI EPOLLPRI + EPOLLOUT = 0x004, +#define EPOLLOUT EPOLLOUT + EPOLLRDNORM = 0x040, +#define EPOLLRDNORM EPOLLRDNORM + EPOLLRDBAND = 0x080, +#define EPOLLRDBAND EPOLLRDBAND + EPOLLWRNORM = 0x100, +#define EPOLLWRNORM EPOLLWRNORM + EPOLLWRBAND = 0x200, +#define EPOLLWRBAND EPOLLWRBAND + EPOLLMSG = 0x400, +#define EPOLLMSG EPOLLMSG + EPOLLERR = 0x008, +#define EPOLLERR EPOLLERR + EPOLLHUP = 0x010, +#define EPOLLHUP EPOLLHUP + EPOLLRDHUP = 0x2000, +#define EPOLLRDHUP EPOLLRDHUP + EPOLLEXCLUSIVE = 1u << 28, +#define EPOLLEXCLUSIVE EPOLLEXCLUSIVE + EPOLLWAKEUP = 1u << 29, +#define EPOLLWAKEUP EPOLLWAKEUP + EPOLLONESHOT = 1u << 30, +#define EPOLLONESHOT EPOLLONESHOT + EPOLLET = 1u << 31 +#define EPOLLET EPOLLET + }; + +/* Valid opcodes ( "op" parameter ) to issue to epoll_ctl(). */ +#define EPOLL_CTL_ADD 1 /* Add a file descriptor to the interface. */ +#define EPOLL_CTL_DEL 2 /* Remove a file descriptor from the interface. */ +#define EPOLL_CTL_MOD 3 /* Change file descriptor epoll_event structure. */ + + +typedef union epoll_data +{ + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; +} epoll_data_t; + +struct epoll_event +{ + uint32_t events; /* Epoll events */ + epoll_data_t data; /* User data variable */ +}; + +int epoll_create(int size); +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); +int epoll_close(int epfd); + +#endif // __APPLE__ + +#ifdef __cplusplus +} +#endif + +#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 9383ae48dc5c6151aea0d9e8a2641603b63da144..8312b74a5015aeb0991b303f0ea8f207cb2cf3eb 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -62,6 +62,7 @@ extern "C" { #include "osMemory.h" #include "osRand.h" #include "osSemphone.h" +#include "osSignal.h" #include "osSocket.h" #include "osString.h" #include "osSysinfo.h" diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 1461ec6d3b177c300e667d93043308d93ea67d0f..7c206afe7a12e9be17ad997ee50625a0cce438e9 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -75,23 +75,29 @@ extern "C" { #define TAOS_OS_FUNC_FILE_SENDIFLE #define TAOS_OS_FUNC_SEMPHONE - #define tsem_t dispatch_semaphore_t - int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value); - int tsem_wait(dispatch_semaphore_t *sem); - int tsem_post(dispatch_semaphore_t *sem); - int tsem_destroy(dispatch_semaphore_t *sem); + typedef struct tsem_s *tsem_t; + int tsem_init(tsem_t *sem, int pshared, unsigned int value); + int tsem_wait(tsem_t *sem); + int tsem_post(tsem_t *sem); + int tsem_destroy(tsem_t *sem); #define TAOS_OS_FUNC_SOCKET_SETSOCKETOPT #define TAOS_OS_FUNC_STRING_STR2INT64 #define TAOS_OS_FUNC_SYSINFO #define TAOS_OS_FUNC_TIMER +#define TAOS_OS_FUNC_SEMPHONE_PTHREAD // specific #define htobe64 htonll typedef int(*__compar_fn_t)(const void *, const void *); // for send function in tsocket.c +#if defined(MSG_NOSIGNAL) +#undef MSG_NOSIGNAL +#endif + #define MSG_NOSIGNAL 0 + #define SO_NO_CHECK 0x1234 #define SOL_TCP 0x1234 #define TCP_KEEPIDLE 0x1234 @@ -100,6 +106,20 @@ typedef int(*__compar_fn_t)(const void *, const void *); #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE #endif +#define TAOS_OS_FUNC_PTHREAD_RWLOCK + +int64_t tsosStr2int64(char *str); + +#include "eok.h" + +void taos_block_sigalrm(void); + +#define TAOS_OS_DEF_EPOLL + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index d718bef6da42f59c5ee4ea7bfaad098b47062984..bbe0f98ec0b632deb39691f74350bced6d2a6515 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -20,8 +20,10 @@ extern "C" { #endif -#ifndef STDERR_FILENO -#define STDERR_FILENO (2) +#ifndef WINDOWS + #ifndef O_BINARY + #define O_BINARY 0 + #endif #endif #define FD_VALID(x) ((x) > STDERR_FILENO) @@ -81,6 +83,20 @@ extern "C" { } \ } while (0) +#define DEFAULT_DOUBLE_COMP(x, y) \ + do { \ + if (isnan(x) && isnan(y)) { return 0; } \ + if (isnan(x)) { return -1; } \ + if (isnan(y)) { return 1; } \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y) + #define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) // align to 8bytes @@ -90,11 +106,11 @@ extern "C" { #ifdef _ISOC11_SOURCE #define threadlocal _Thread_local #elif defined(__APPLE__) - #define threadlocal + #define threadlocal __thread #elif defined(__GNUC__) && !defined(threadlocal) #define threadlocal __thread #else - #define threadlocal + #define threadlocal __declspec( thread ) #endif #ifdef __cplusplus diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index 67cfdb3b530a7db17b834a1443d681e1cd5361a7..540efc310c1d89530d6e41733ce2419ad79b1e36 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -25,8 +25,8 @@ extern "C" { // TAOS_OS_FUNC_DIR void taosRemoveDir(char *rootDir); int taosMkDir(const char *pathname, mode_t mode); -void taosRename(char* oldName, char *newName); void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays); +int32_t taosRename(char* oldName, char *newName); int32_t taosCompressFile(char *srcFileName, char *destFileName); #ifdef __cplusplus diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index c9b3b9cd76d675169746a2478bfbd775d6da190a..c117ae4039d05b6a063726ec9052fe5f5562365e 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,10 +20,13 @@ extern "C" { #endif +#include "osSocket.h" + int64_t taosReadImp(int32_t fd, void *buf, int64_t count); int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath); +int64_t taosCopy(char *from, char *to); #define taosRead(fd, buf, count) taosReadImp(fd, buf, count) #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count) @@ -37,7 +40,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h index 0616006650eb2d53cdf7bd70b68c9f60748deac5..2cf7e14d2f4bc9fc124cdf4de167c5b2cb93f4bb 100644 --- a/src/os/inc/osMemory.h +++ b/src/os/inc/osMemory.h @@ -35,7 +35,7 @@ void taosDumpMemoryLeak(); void * taosTMalloc(size_t size); void * taosTCalloc(size_t nmemb, size_t size); void * taosTRealloc(void *ptr, size_t size); -void taosTZfree(void *ptr); +void * taosTZfree(void *ptr); size_t taosTSizeof(void *ptr); void taosTMemset(void *ptr, int c); @@ -52,7 +52,7 @@ void taosTMemset(void *ptr, int c); free((void *)(x)); \ x = 0; \ } \ - } while (0); + } while (0) #ifdef TAOS_MEM_CHECK #ifdef TAOS_MEM_CHECK_TEST diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index a71e74e97f4d9910414a5e801b89a1968d1df050..3332a9234b040aaa49c1d097e63f03a6c9bde25b 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -28,13 +28,29 @@ extern "C" { #define tsem_destroy sem_destroy #endif +#ifdef TAOS_OS_FUNC_PTHREAD_RWLOCK + #define pthread_rwlock_t pthread_mutex_t + #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL) + #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock) + #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock) + #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock) + #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock) + + #define pthread_spinlock_t pthread_mutex_t + #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL) + #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock) + #define pthread_spin_lock(lock) pthread_mutex_lock(lock) + #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock) +#endif + // TAOS_OS_FUNC_SEMPHONE_PTHREAD -bool taosCheckPthreadValid(pthread_t thread); -int64_t taosGetPthreadId(); -void taosResetPthread(pthread_t *thread); -bool taosComparePthread(pthread_t first, pthread_t second); +bool taosCheckPthreadValid(pthread_t thread); +int64_t taosGetSelfPthreadId(); +int64_t taosGetPthreadId(pthread_t thread); +void taosResetPthread(pthread_t* thread); +bool taosComparePthread(pthread_t first, pthread_t second); int32_t taosGetPId(); -int32_t taosGetCurrentAPPName(char *name, int32_t* len); +int32_t taosGetCurrentAPPName(char* name, int32_t* len); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMgmt.h b/src/os/inc/osSignal.h similarity index 50% rename from src/dnode/inc/dnodeMgmt.h rename to src/os/inc/osSignal.h index 2038ef5286b32522b11409ba5a253b33228b984d..c13cd83178691fa8c617369904cb37a20ab27479 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/os/inc/osSignal.h @@ -13,35 +13,48 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_DNODE_MGMT_H -#define TDENGINE_DNODE_MGMT_H +#ifndef TDENGINE_OS_SIGNAL_H +#define TDENGINE_OS_SIGNAL_H #ifdef __cplusplus extern "C" { #endif -#include "trpc.h" +#include "os.h" +#include "taosdef.h" +#include -int32_t dnodeInitMgmt(); -void dnodeCleanupMgmt(); -int32_t dnodeInitMgmtTimer(); -void dnodeCleanupMgmtTimer(); -void dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg); +#ifndef SIGALRM + #define SIGALRM 1234 +#endif + +#ifndef SIGHUP + #define SIGHUP 1230 +#endif -void* dnodeGetVnode(int32_t vgId); -int32_t dnodeGetVnodeStatus(void *pVnode); -void* dnodeGetVnodeRworker(void *pVnode); -void* dnodeGetVnodeWworker(void *pVnode); -void* dnodeGetVnodeWal(void *pVnode); -void* dnodeGetVnodeTsdb(void *pVnode); -void dnodeReleaseVnode(void *pVnode); +#ifndef SIGCHLD + #define SIGCHLD 1234 +#endif -void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); -void dnodeGetEpSetForPeer(SRpcEpSet *epSet); -void dnodeGetEpSetForShell(SRpcEpSet *epSet); +#ifndef SIGUSR1 + #define SIGUSR1 1234 +#endif + +#ifndef SIGUSR2 + #define SIGUSR2 1234 +#endif + +#ifndef SIGBREAK + #define SIGBREAK 1234 +#endif + +typedef void (*FSignalHandler)(int32_t signum, void *sigInfo, void *context); +void taosSetSignal(int32_t signum, FSignalHandler sigfp); +void taosIgnSignal(int32_t signum); +void taosDflSignal(int32_t signum); #ifdef __cplusplus } #endif -#endif +#endif // TDENGINE_TTIME_H diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index baf7687dd03e1ee4f6dd92e3204244b3d31b7a1f..111fca712c20c1b0ef4b2c19e0a30f61c5e2029e 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -33,11 +33,13 @@ extern "C" { x = FD_INITIALIZER; \ } \ } - typedef int32_t SOCKET; #endif #ifndef TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) taosCloseSocket(pollFd) #endif #ifdef TAOS_RANDOM_NETWORK_FAIL @@ -59,7 +61,9 @@ extern "C" { // TAOS_OS_FUNC_SOCKET int32_t taosSetNonblocking(SOCKET sock, int32_t on); +void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); +void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); diff --git a/src/os/inc/osString.h b/src/os/inc/osString.h index b2846a31bccfc61865d5729d93d48a484e0eb9b7..97966ed412d0ae4ef4970de4dbb6d48c82130671 100644 --- a/src/os/inc/osString.h +++ b/src/os/inc/osString.h @@ -39,7 +39,7 @@ extern "C" { do { \ strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ - } while (0); + } while (0) #ifndef TAOS_OS_FUNC_STRING_STR2INT64 int64_t tsosStr2int64(char *str); @@ -47,7 +47,7 @@ extern "C" { // USE_LIBICONV int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); -bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len); +bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int *len); int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes); bool taosValidateEncodec(const char *encodec); char * taosCharsetReplace(char *charsetstr); diff --git a/src/os/inc/osSysinfo.h b/src/os/inc/osSysinfo.h index 2b98f8b4bfa36e2bb5f8c5a3d2a595df2aba74ea..25c9c97b1e888d9f4371eea67a16efea728294a4 100644 --- a/src/os/inc/osSysinfo.h +++ b/src/os/inc/osSysinfo.h @@ -21,10 +21,16 @@ extern "C" { #endif // TAOS_OS_FUNC_SYSINFO +typedef struct { + int64_t tsize; + int64_t avail; +} SysDiskSize; + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize); void taosGetSystemInfo(); bool taosGetProcIO(float *readKB, float *writeKB); bool taosGetBandSpeed(float *bandSpeedKb); -bool taosGetDisk(); +void taosGetDisk(); bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ; bool taosGetProcMemory(float *memoryUsedMB) ; bool taosGetSysMemory(float *memoryUsedMB); @@ -32,6 +38,7 @@ void taosPrintOsInfo(); int taosSystem(const char * cmd) ; void taosKillSystem(); bool taosGetSystemUid(char *uid); +char *taosGetCmdlineByPID(int pid); // TAOS_OS_FUNC_SYSINFO_CORE void taosSetCoreDump(); diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 5a1e6425728b88a5461ce3fe83a060e23161ae86..a058f2cc99029e311168428ce6e91f88212a4c73 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -46,6 +46,8 @@ #include "msvcFcntl.h" #include "msvcLibgen.h" #include "msvcStdio.h" +#include "msvcUnistd.h" +#include "msvcLibgen.h" #include "sys/msvcStat.h" #include "sys/msvcTypes.h" @@ -67,6 +69,8 @@ extern "C" { #define TAOS_OS_FUNC_FILE_GETTMPFILEPATH #define TAOS_OS_FUNC_FILE_FTRUNCATE +#define TAOS_OS_FUNC_DIR + #define TAOS_OS_FUNC_MATH #define SWAP(a, b, c) \ do { \ @@ -93,6 +97,12 @@ typedef SOCKET eventfd_t; #define TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 100 + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif #define TAOS_OS_DEF_ZU #define PRIzu "ld" @@ -138,7 +148,6 @@ typedef int (*__compar_fn_t)(const void *, const void *); #define in_addr_t unsigned long #define socklen_t int #define htobe64 htonll -#define getpid _getpid struct tm *localtime_r(const time_t *timep, struct tm *result); char * strptime(const char *buf, const char *fmt, struct tm *tm); @@ -147,15 +156,8 @@ char * getpass(const char *prefix); int flock(int fd, int option); int fsync(int filedes); char * strndup(const char *s, size_t n); -char * dirname(char *pszPathname); int gettimeofday(struct timeval *ptv, void *pTimeZone); -// for access function in io.h -#define F_OK 00 //Existence only -#define W_OK 02 //Write - only -#define R_OK 04 //Read - only -#define X_OK 06 //Read and write - // for send function in tsocket.c #define MSG_NOSIGNAL 0 #define SO_NO_CHECK 0x1234 @@ -191,23 +193,23 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); #define PATH_MAX 256 #endif -//for signal, not dispose -#define SIGALRM 1234 -typedef int sigset_t; -struct sigaction { - void (*sa_handler)(int); -}; -int sigaction(int, struct sigaction *, void *); +#define TAOS_OS_FUNC_SIGNAL typedef struct { int we_wordc; - char **we_wordv; + char *we_wordv[1]; int we_offs; - char wordPos[20]; + char wordPos[1025]; } wordexp_t; -int wordexp(const char *words, wordexp_t *pwordexp, int flags); +int wordexp(char *words, wordexp_t *pwordexp, int flags); void wordfree(wordexp_t *pwordexp); +#define openlog(a, b, c) +#define closelog() +#define LOG_ERR 0 +#define LOG_INFO 1 +void syslog(int unused, const char *format, ...); + #define TAOS_OS_FUNC_ATOMIC #define atomic_load_8(ptr) (*(char volatile*)(ptr)) #define atomic_load_16(ptr) (*(short volatile*)(ptr)) diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 6adebabec01ae378a4c5f3d82389508334210972..8ac97eacfbc53eb67fa141f8bdddc79d8a65777c 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -17,16 +17,27 @@ #include "os.h" #include "tglobal.h" +static const char* expand_like_shell(const char *path) { + static __thread char buf[TSDB_FILENAME_LEN]; + buf[0] = '\0'; + wordexp_t we; + if (wordexp(path, &we, 0)) return "/tmp/taosd"; + if (sizeof(buf)<=snprintf(buf, sizeof(buf), "%s", we.we_wordv[0])) return "/tmp/taosd"; + wordfree(&we); + return buf; +} + void osInit() { if (configDir[0] == 0) { - strcpy(configDir, "~/TDengine/cfg"); + strcpy(configDir, expand_like_shell("/usr/local/etc/taos")); } + strcpy(tsDataDir, expand_like_shell("/usr/local/var/lib/taos")); + strcpy(tsLogDir, expand_like_shell("/usr/local/var/log/taos")); + strcpy(tsScriptDir, expand_like_shell("/usr/local/etc/taos")); strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); - strcpy(tsDataDir, "~/TDengine/data"); - strcpy(tsLogDir, "~/TDengine/log"); - strcpy(tsScriptDir, "~/TDengine/cfg"); strcpy(tsOsName, "Darwin"); } + diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index dacf4db74137b8cc58db2b390ba971cf9dfbc012..4236ea1c5f08c8b8a8a81ae10c7b442ad3b5ce6a 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -17,41 +17,49 @@ #include "os.h" #include "tulog.h" -#define _SEND_FILE_STEP_ 1000 - int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { - fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = {0}; - - for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); - if (rlen <= 0) { - return writeLen; - } else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); - return (int)(writeLen + rlen); - } else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); - writeLen += _SEND_FILE_STEP_; - } + int r = 0; + if (offset) { + r = fseek(in_file, *offset, SEEK_SET); + if (r==-1) return -1; } - - int remain = count - writeLen; - if (remain > 0) { - size_t rlen = fread(buffer, 1, remain, in_file); - if (rlen <= 0) { - return writeLen; - } else { - fwrite(buffer, 1, remain, out_file); - writeLen += remain; + off_t len = count; + while (len>0) { + char buf[1024*16]; + off_t n = sizeof(buf); + if (len0) { + char buf[1024*16]; + off_t n = sizeof(buf); + if (len. */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" -int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value) { - *sem = dispatch_semaphore_create(value); - if (*sem == NULL) { +#include + +// #define SEM_USE_PTHREAD +// #define SEM_USE_POSIX +#define SEM_USE_SEM + +#ifdef SEM_USE_SEM +#include +#include +#include +#include +#include + +static pthread_t sem_thread; +static pthread_once_t sem_once; +static task_t sem_port; +static volatile int sem_inited = 0; +static semaphore_t sem_exit; + +static void* sem_thread_routine(void *arg) { + (void)arg; + sem_port = mach_task_self(); + kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__); + sem_inited = -1; + return NULL; + } + sem_inited = 1; + semaphore_wait(sem_exit); + return NULL; +} + +static void once_init(void) { + int r = 0; + r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__); + return; + } + while (sem_inited==0) { + ; + } +} +#endif + +struct tsem_s { +#ifdef SEM_USE_PTHREAD + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int64_t val; +#elif defined(SEM_USE_POSIX) + size_t id; + sem_t *sem; +#elif defined(SEM_USE_SEM) + semaphore_t sem; +#else // SEM_USE_PTHREAD + dispatch_semaphore_t sem; +#endif // SEM_USE_PTHREAD + + volatile unsigned int valid:1; +}; + +int tsem_init(tsem_t *sem, int pshared, unsigned int value) { + // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + if (*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = (struct tsem_s*)calloc(1, sizeof(*p)); + if (!p) { + fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + +#ifdef SEM_USE_PTHREAD + int r = pthread_mutex_init(&p->lock, NULL); + do { + if (r) break; + r = pthread_cond_init(&p->cond, NULL); + if (r) { + pthread_mutex_destroy(&p->lock); + break; + } + p->val = value; + } while (0); + if (r) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + static size_t tick = 0; + do { + size_t id = atomic_add_fetch_64(&tick, 1); + if (id==SEM_VALUE_MAX) { + atomic_store_64(&tick, 0); + id = 0; + } + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", id); + p->sem = sem_open(name, O_CREAT|O_EXCL, pshared, value); + p->id = id; + if (p->sem!=SEM_FAILED) break; + int e = errno; + if (e==EEXIST) continue; + if (e==EINTR) continue; + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } while (p->sem==SEM_FAILED); +#elif defined(SEM_USE_SEM) + pthread_once(&sem_once, once_init); + if (sem_inited!=1) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem); + errno = ENOMEM; return -1; - } else { - return 0; } + kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem); + // we fail-fast here, because we have less-doc about semaphore_create for the moment + abort(); + } +#else // SEM_USE_PTHREAD + p->sem = dispatch_semaphore_create(value); + if (p->sem == NULL) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#endif // SEM_USE_PTHREAD + + p->valid = 1; + + *sem = p; + + return 0; } -int tsem_wait(dispatch_semaphore_t *sem) { - dispatch_semaphore_wait(*sem, DISPATCH_TIME_FOREVER); +int tsem_wait(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val -= 1; + if (p->val < 0) { + if (pthread_cond_wait(&p->cond, &p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } return 0; +#elif defined(SEM_USE_POSIX) + return sem_wait(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_wait(p->sem); +#else // SEM_USE_PTHREAD + return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); +#endif // SEM_USE_PTHREAD } -int tsem_post(dispatch_semaphore_t *sem) { - dispatch_semaphore_signal(*sem); +int tsem_post(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val += 1; + if (p->val <= 0) { + if (pthread_cond_signal(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } return 0; +#elif defined(SEM_USE_POSIX) + return sem_post(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_signal(p->sem); +#else // SEM_USE_PTHREAD + return dispatch_semaphore_signal(p->sem); +#endif // SEM_USE_PTHREAD } -int tsem_destroy(dispatch_semaphore_t *sem) { +int tsem_destroy(tsem_t *sem) { + // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + if (!*sem) { + // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); + return 0; + } + struct tsem_s *p = *sem; + if (!p->valid) { + // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); + return 0; + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->valid = 0; + if (pthread_cond_destroy(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_destroy(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", p->id); + int r = sem_unlink(name); + if (r) { + int e = errno; + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } +#elif defined(SEM_USE_SEM) + semaphore_destroy(sem_port, p->sem); +#else // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD + + p->valid = 0; + free(p); + + *sem = NULL; return 0; } + +bool taosCheckPthreadValid(pthread_t thread) { + uint64_t id = 0; + int r = pthread_threadid_np(thread, &id); + return r ? false : true; +} + +int64_t taosGetSelfPthreadId() { + uint64_t id; + pthread_threadid_np(0, &id); + return (int64_t) id; +} + +int64_t taosGetPthreadId(pthread_t thread) { + return (int64_t)thread; +} + +void taosResetPthread(pthread_t* thread) { + *thread = NULL; +} + +bool taosComparePthread(pthread_t first, pthread_t second) { + return pthread_equal(first, second) ? true : false; +} + +int32_t taosGetPId() { + return (int32_t)getpid(); +} + +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { + char buf[PATH_MAX+1]; + buf[0] = '\0'; + proc_name(getpid(), buf, sizeof(buf)-1); + buf[PATH_MAX] = '\0'; + size_t n = strlen(buf); + if (len) *len = n; + if (name) strcpy(name, buf); + return 0; +} + + diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index f8aa02dcffd14e2def6339ccb507b9f9ab2489d1..6af6285f56aa33200379aa52f94e0ab39de80644 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -18,44 +18,140 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" +#include "taoserror.h" +#include +#include + static void taosGetSystemTimezone() { - // get and set default timezone SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); + if (cfg_timezone == NULL) return; + if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) { + return; + } + + /* load time zone string from /etc/localtime */ + char buf[4096]; + char *tz = NULL; { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n<0) { + uError("read /etc/localtime error, reason:%s", strerror(errno)); + return; } - else { - strcpy(tsTimezone, tz); + buf[n] = '\0'; + for(int i=n-1; i>=0; --i) { + if (buf[i]=='/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } } - cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("timezone not configured, use default"); + if (!tz || 0==strchr(tz, '/')) { + uError("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", + tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone/3600); + + // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uWarn("timezone not configured, set to system default:%s", tsTimezone); } -static void taosGetSystemLocale() { - // get and set default locale +/* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ +static void taosGetSystemLocale() { // get and set default locale + char sep = '.'; + char *locale = NULL; + SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(tsLocale, "en_US.UTF-8"); + } else { tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("locale not configured, set to default:%s", tsLocale); + uWarn("locale not configured, set to system default:%s", tsLocale); } } + /* if user does not specify the charset, extract it from locale */ SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - strcpy(tsCharset, "cp936"); - cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("charset not configured, set to default:%s", tsCharset); + char *str = strrchr(tsLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); + + free(revisedCharset); + uWarn("charset not configured, set to system default:%s", tsCharset); + } else { + strcpy(tsCharset, "UTF-8"); + uWarn("can't get locale and charset from system, set it to UTF-8"); + } } } -void taosPrintOsInfo() {} +void taosPrintOsInfo() { + uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize / 1024); + // uInfo(" os openMax: %" PRId64, tsOpenMax); + // uInfo(" os streamMax: %" PRId64, tsStreamMax); + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + + struct utsname buf; + if (uname(&buf)) { + uInfo(" can't fetch os info"); + return; + } + uInfo(" os sysname: %s", buf.sysname); + uInfo(" os nodename: %s", buf.nodename); + uInfo(" os release: %s", buf.release); + uInfo(" os version: %s", buf.version); + uInfo(" os machine: %s", buf.machine); + uInfo("=================================="); +} void taosKillSystem() { uError("function taosKillSystem, exit!"); @@ -63,12 +159,26 @@ void taosKillSystem() { } void taosGetSystemInfo() { + // taosGetProcInfos(); + + tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); + long physical_pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); + tsPageSize = page_size; + + // float tmp1, tmp2; + // taosGetSysMemory(&tmp1); + // taosGetProcMemory(&tmp2); + // taosGetDisk(); + // taosGetBandSpeed(&tmp1); + // taosGetCpuUsage(&tmp1, &tmp2); + // taosGetProcIO(&tmp1, &tmp2); + taosGetSystemTimezone(); taosGetSystemLocale(); } -bool taosGetDisk() { return true; } - bool taosGetProcIO(float *readKB, float *writeKB) { *readKB = 0; *writeKB = 0; @@ -101,4 +211,39 @@ int taosSystem(const char *cmd) { return -1; } -void taosSetCoreDump() {} \ No newline at end of file +void taosSetCoreDump() {} + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { + struct statvfs info; + if (statvfs(tsDataDir, &info)) { + uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } else { + diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->avail = info.f_bavail * info.f_frsize; + return 0; + } +} + +char cmdline[1024]; + +char *taosGetCmdlineByPID(int pid) { + errno = 0; + + if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { + fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); + return strerror(errno); + } + + return cmdline; +} + +bool taosGetSystemUid(char *uid) { + uuid_t uuid = {0}; + uuid_generate(uuid); + // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null + uuid_unparse_lower(uuid, uid); + return true; +} + diff --git a/src/os/src/darwin/darwinTimer.c b/src/os/src/darwin/darwinTimer.c index 5fe65fb99ed72357d88ecbf5d4699bcbb42360c6..ee1becc91af5237296bb2a562982b33cc5c78f10 100644 --- a/src/os/src/darwin/darwinTimer.c +++ b/src/os/src/darwin/darwinTimer.c @@ -13,9 +13,82 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" +#if 1 +#include + +static void (*timer_callback)(int); +static int timer_ms = 0; +static pthread_t timer_thread; +static int timer_kq = -1; +static volatile int timer_stop = 0; + +static void* timer_routine(void *arg) { + (void)arg; + + int r = 0; + struct timespec to = {0}; + to.tv_sec = timer_ms / 1000; + to.tv_nsec = (timer_ms % 1000) * 1000000; + while (!timer_stop) { + struct kevent64_s kev[10] = {0}; + r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); + if (r!=0) { + fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); + abort(); + } + timer_callback(SIGALRM); // just mock + } + + return NULL; +} + +int taosInitTimer(void (*callback)(int), int ms) { + int r = 0; + timer_ms = ms; + timer_callback = callback; + + timer_kq = kqueue(); + if (timer_kq==-1) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + + r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + return 0; +} + +void taosUninitTimer() { + int r = 0; + timer_stop = 1; + r = pthread_join(timer_thread, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + close(timer_kq); + timer_kq = -1; +} + +void taos_block_sigalrm(void) { + // we don't know if there's any specific API for SIGALRM to deliver to specific thread + // this implementation relies on kqueue rather than SIGALRM +} +#else int taosInitTimer(void (*callback)(int), int ms) { signal(SIGALRM, callback); @@ -34,3 +107,17 @@ void taosUninitTimer() { setitimer(ITIMER_REAL, &tv, NULL); } +void taos_block_sigalrm(void) { + // since SIGALRM has been used + // consideration: any better solution? + static __thread int already_set = 0; + if (!already_set) { + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGALRM); + pthread_sigmask(SIG_BLOCK, &set, NULL); + already_set = 1; + } +} +#endif + diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c new file mode 100644 index 0000000000000000000000000000000000000000..f731aaf330cb8618aa8a2dc51401ac45ecdc5440 --- /dev/null +++ b/src/os/src/darwin/eok.c @@ -0,0 +1,893 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + +#include "eok.h" + +#include "os.h" + +#include + +// #define BALANCE_CHECK_WHEN_CLOSE + +#ifdef ENABLE_LOG +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) +#else // !ENABLE_LOG +#define D(fmt, ...) (void)fmt +#define E(fmt, ...) (void)fmt +#endif // ENABLE_LOG + +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +static int eok_dummy = 0; + +typedef struct ep_over_kq_s ep_over_kq_t; +typedef struct eok_event_s eok_event_t; + +struct ep_over_kq_s { + int kq; + + // !!! + // idx in the eoks list + // used as pseudo-file-desciptor + // must be 'closed' with epoll_close + int idx; + + ep_over_kq_t *next; + + int sv[2]; // 0 for read, 1 for write + + // all registered 'epoll events, key by fd' + int evs_count; + eok_event_t *evs_head; + eok_event_t *evs_tail; + eok_event_t *evs_free; + + // all kev changes list pending to be processed by kevent64 + // key by tuple (ident,filter), ident === fd in this case + struct kevent64_s *kchanges; + int nchanges; + int ichanges; + + // kev eventslist for kevent64 to store active events + // they remain alive among kevent64 calls + struct kevent64_s *kevslist; + int nevslist; + + pthread_mutex_t lock; + + volatile unsigned int lock_valid:1; + volatile unsigned int waiting:1; + volatile unsigned int changed:1; + volatile unsigned int wakenup:1; + volatile unsigned int stopping:1; +}; + +struct eok_event_s { + int fd; + struct epoll_event epev; + volatile unsigned int changed; // 0:registered;1:add;2:mod;3:del + + eok_event_t *next; + eok_event_t *prev; +}; + +typedef struct eoks_s eoks_t; +struct eoks_s { + pthread_mutex_t lock; + ep_over_kq_t **eoks; // note: this memory leaks when process terminates + int neoks; // we can add an extra api to let user clean + ep_over_kq_t *eoks_free_list; // currently, we just keep it simple stupid +}; + +static eoks_t eoks = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .eoks = NULL, + .neoks = 0, + .eoks_free_list = NULL, +}; + +#ifdef ENABLE_LOG +static const char* op_str(int op) { + switch (op) { + case EPOLL_CTL_ADD: return "EPOLL_CTL_ADD"; + case EPOLL_CTL_MOD: return "EPOLL_CTL_MOD"; + case EPOLL_CTL_DEL: return "EPOLL_CTL_DEL"; + default: return "UNKNOWN"; + } +} + +static __thread char buf_slots[10][1024] = {0}; +static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); +static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); + +static const char* events_str(uint32_t events, int slots) { + A(slots>=0 && slots on linux + // EPOLLIN = 0x001, + // #define EPOLLIN EPOLLIN + // EPOLLPRI = 0x002, + // #define EPOLLPRI EPOLLPRI + // EPOLLOUT = 0x004, + // #define EPOLLOUT EPOLLOUT + // EPOLLRDNORM = 0x040, + // #define EPOLLRDNORM EPOLLRDNORM + // EPOLLRDBAND = 0x080, + // #define EPOLLRDBAND EPOLLRDBAND + // EPOLLWRNORM = 0x100, + // #define EPOLLWRNORM EPOLLWRNORM + // EPOLLWRBAND = 0x200, + // #define EPOLLWRBAND EPOLLWRBAND + // EPOLLMSG = 0x400, + // #define EPOLLMSG EPOLLMSG + // EPOLLERR = 0x008, + // #define EPOLLERR EPOLLERR + // EPOLLHUP = 0x010, + // #define EPOLLHUP EPOLLHUP + // EPOLLRDHUP = 0x2000, + // #define EPOLLRDHUP EPOLLRDHUP + // EPOLLEXCLUSIVE = 1u << 28, + // #define EPOLLEXCLUSIVE EPOLLEXCLUSIVE + // EPOLLWAKEUP = 1u << 29, + // #define EPOLLWAKEUP EPOLLWAKEUP + // EPOLLONESHOT = 1u << 30, + // #define EPOLLONESHOT EPOLLONESHOT + // EPOLLET = 1u << 31 + // #define EPOLLET EPOLLET +#define CHK_EV(ev) \ + if (len>0 && (events & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EPOLLIN); + CHK_EV(EPOLLPRI); + CHK_EV(EPOLLOUT); + CHK_EV(EPOLLRDNORM); + CHK_EV(EPOLLRDBAND); + CHK_EV(EPOLLWRNORM); + CHK_EV(EPOLLWRBAND); + CHK_EV(EPOLLMSG); + CHK_EV(EPOLLERR); + CHK_EV(EPOLLHUP); + CHK_EV(EPOLLRDHUP); + CHK_EV(EPOLLEXCLUSIVE); + CHK_EV(EPOLLWAKEUP); + CHK_EV(EPOLLONESHOT); + CHK_EV(EPOLLET); +#undef CHK_EV + return buf; +} + +static const char* kev_flags_str(uint16_t flags, int slots) { + A(slots>=0 && slots + // #define EV_ADD 0x0001 /* add event to kq (implies enable) */ + // #define EV_DELETE 0x0002 /* delete event from kq */ + // #define EV_ENABLE 0x0004 /* enable event */ + // #define EV_DISABLE 0x0008 /* disable event (not reported) */ + // /* flags */ + // #define EV_ONESHOT 0x0010 /* only report one occurrence */ + // #define EV_CLEAR 0x0020 /* clear event state after reporting */ + // #define EV_RECEIPT 0x0040 /* force immediate event output */ + // /* ... with or without EV_ERROR */ + // /* ... use KEVENT_FLAG_ERROR_EVENTS */ + // /* on syscalls supporting flags */ + // #define EV_DISPATCH 0x0080 /* disable event after reporting */ + // #define EV_UDATA_SPECIFIC 0x0100 /* unique kevent per udata value */ + // #define EV_DISPATCH2 (EV_DISPATCH | EV_UDATA_SPECIFIC) + // /* ... in combination with EV_DELETE */ + // /* will defer delete until udata-specific */ + // /* event enabled. EINPROGRESS will be */ + // /* returned to indicate the deferral */ + // #define EV_VANISHED 0x0200 /* report that source has vanished */ + // /* ... only valid with EV_DISPATCH2 */ + // #define EV_SYSFLAGS 0xF000 /* reserved by system */ + // #define EV_FLAG0 0x1000 /* filter-specific flag */ + // #define EV_FLAG1 0x2000 /* filter-specific flag */ + // /* returned values */ + // #define EV_EOF 0x8000 /* EOF detected */ + // #define EV_ERROR 0x4000 /* error, data contains errno */ +#define CHK_EV(ev) \ + if (len>0 && (flags & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EV_ADD); + CHK_EV(EV_DELETE); + CHK_EV(EV_ENABLE); + CHK_EV(EV_DISABLE); + CHK_EV(EV_ONESHOT); + CHK_EV(EV_CLEAR); + CHK_EV(EV_RECEIPT); + CHK_EV(EV_DISPATCH); + CHK_EV(EV_UDATA_SPECIFIC); + CHK_EV(EV_DISPATCH2); + CHK_EV(EV_VANISHED); + CHK_EV(EV_SYSFLAGS); + CHK_EV(EV_FLAG0); + CHK_EV(EV_FLAG1); + CHK_EV(EV_EOF); + CHK_EV(EV_ERROR); +#undef CHK_EV + return buf; +} +#endif // ENABLE_LOG + +static ep_over_kq_t* eoks_alloc(void); +static void eoks_free(ep_over_kq_t *eok); +static ep_over_kq_t* eoks_find(int epfd); + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd); +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok); +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev); +static void eok_wakeup(ep_over_kq_t *eok); + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev); + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents); + +int epoll_create(int size) { + (void)size; + int e = 0; + ep_over_kq_t *eok = eoks_alloc(); + if (!eok) return -1; + + A(eok->kq==-1, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); + A(eok->sv[0]==-1, "internal logic error"); + A(eok->sv[1]==-1, "internal logic error"); + + eok->kq = kqueue(); + if (eok->kq==-1) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, eok->sv)) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &eok_dummy; + D("epoll_create epfd:[%d] and sv0[%d]", eok->idx, eok->sv[0]); + if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { + e = errno; + epoll_close(eok->idx); + errno = e; + return -1; + } + + return eok->idx; +} + +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { + D("epoll_ctling epfd:[%d], op:[%s], fd:[%d], events:[%04x:%s]", + epfd, op_str(op), fd, + event ? event->events : 0, + event ? events_str(event->events, 0) : "NULL"); + int e = 0; + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + if (fd==-1) { + errno = EBADF; + return -1; + } + if (event && !(event->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP | EPOLLOUT))) { + e = ENOTSUP; + return -1; + } + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + eok_event_t* oev = eok_find_ev(eok, fd); + if (op==EPOLL_CTL_ADD && oev) { + e = EEXIST; + break; + } + if (op!=EPOLL_CTL_ADD && !oev) { + e = ENOENT; + break; + } + if (op!=EPOLL_CTL_DEL && !event) { + e = EINVAL; + break; + } + + // prepare krev/kwev + struct kevent64_s krev = {0}; + struct kevent64_s kwev = {0}; + krev.ident = -1; + kwev.ident = -1; + uint16_t flags = 0; + // prepare internal eok event + eok_event_t ev = {0}; + ev.fd = fd; + if (event) ev.epev = *event; + struct epoll_event *pev = event; + switch (op) { + case EPOLL_CTL_ADD: { + flags = EV_ADD; + ev.changed = 1; + } break; + case EPOLL_CTL_MOD: { + flags = EV_ADD; + ev.changed = 2; + } break; + case EPOLL_CTL_DEL: { + // event is ignored + // pev points to registered epoll_event + pev = &oev->epev; + flags = EV_DELETE; + ev.changed = 3; + } break; + default: { + e = ENOTSUP; + } break; + } + + if (e) break; + + // udata will be delayed to be set + if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + flags |= EV_EOF; + EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); + } + if (pev->events & EPOLLOUT) { + EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); + } + + // refresh registered evlist and changelist in a transaction way + if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { + e = errno; + A(e, "internal logic error"); + break; + } + eok->changed = 1; + eok_wakeup(eok); + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + return -1; + } + + return 0; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to); + +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { + taos_block_sigalrm(); + + int e = 0; + if (!events) { + errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + return -1; + } + if (maxevents<=0) { + errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + return -1; + } + + struct timespec abstime = {0}; + A(TIME_UTC==timespec_get(&abstime, TIME_UTC), "internal logic error"); + + if (timeout!=-1) { + if (timeout<0) timeout = 0; + int64_t t = abstime.tv_nsec + timeout * 1000000; + abstime.tv_sec += t / 1000000000; + abstime.tv_nsec = t % 1000000000; + } + + int r = 0; + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + errno = EBADF; + return -1; + } + + int cnts = 0; + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + cnts = 0; + A(eok->waiting==0, "internal logic error"); + struct kevent64_s *eventslist = eok_alloc_eventslist(eok, maxevents); + if (!eventslist) { + e = ENOMEM; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + break; + } + memset(eventslist, 0, maxevents * sizeof(*eventslist)); + + struct timespec now = {0}; + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + struct timespec to = do_timespec_diff(&now, &abstime); + struct timespec *pto = &to; + if (timeout==-1) { + pto = NULL; + } + + // taking the changelist + struct kevent64_s *kchanges = eok->kchanges; + int nchanges = eok->nchanges; + int ichanges = eok->ichanges; + // let outside world to add changes + eok->kchanges = NULL; + eok->nchanges = 0; + eok->ichanges = 0; + + eok->changed = 0; + eok->wakenup = 0; + eok->waiting = 1; + + A(0==pthread_mutex_unlock(&eok->lock), ""); + if (ichanges>0) { + D("kevent64 epfd[%d] changing [%d] changes and waiting...", eok->idx, ichanges); + } + errno = 0; + r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); + e = errno; + if (e) { + E("kevent64 epfd[%d] waiting done, with r[%d]", eok->idx, r); + } + A(0==pthread_mutex_lock(&eok->lock), ""); + + eok->waiting = 0; + + if (kchanges) { + if (eok->kchanges==NULL) { + // reuse + A(eok->nchanges==0 && eok->ichanges==0, "internal logic error"); + eok->kchanges = kchanges; + eok->nchanges = nchanges; + } else { + free(kchanges); + kchanges = NULL; + } + nchanges = 0; + ichanges = 0; + } + + if (r==0) { + A(timeout!=-1, "internal logic error"); + } + for (int i=0; iudata && eok->evs_head && eok->evs_tail, "internal logic error"); + eok_event_t *ev = (eok_event_t*)kev->udata; + A(kev->ident == ev->fd, "internal logic error"); + if (kev->flags & EV_ERROR) { + D("epfd[%d] error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", + epfd, ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); + } + switch (kev->filter) { + case EVFILT_READ: { + A((ev->epev.events & EPOLLIN), "internal logic errro"); + if (ev->epev.data.ptr==&eok_dummy) { + // it's coming from wakeup socket pair + char c = '\0'; + A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); + A(0==memcmp(&c, "1", 1), "internal logic error"); + D("epfd[%d] wokenup", epfd); + continue; + } else { + if (ev->changed==3) { + D("epfd[%d] already requested to delete for fd[%d]", epfd, ev->fd); + // TODO: write a unit test for this case + // EV_DELETE? + continue; + } + // converting to epoll_event + // we shall collect all kevents for the uniq fd into one epoll_evnt + // but currently, taos never use EPOLLOUT + // just let it this way for the moment + struct epoll_event pev = {0}; + pev.data.ptr = ev->epev.data.ptr; + pev.events = EPOLLIN; + if (kev->flags & EV_EOF) { + // take all these as EOF for the moment + pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); + } + // rounded to what user care + pev.events = pev.events & ev->epev.events; + D("epfd[%d] events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + epfd, + ev->fd, pev.events, events_str(pev.events, 0), + ev->epev.events, events_str(ev->epev.events, 1), + kev->flags, kev_flags_str(kev->flags, 2)); + // now we get ev and store it + events[cnts++] = pev; + } + } break; + case EVFILT_WRITE: { + A(0, "not implemented yet"); + } break; + default: { + A(0, "internal logic error"); + } break; + } + } + if (r>=0) { + // we can safely rule out delete-requested-events from the regitered evlists + // if only changelist are correctly registered + eok_event_t *p = eok->evs_head; + while (p) { + eok_event_t *next = p->next; + if (p->changed==3) { + D("epfd[%d] removing registered event for fd[%d]: [%04x:%s]", epfd, p->fd, p->epev.events, events_str(p->epev.events, 0)); + eok_free_ev(eok, p); + } + p = next; + } + } + if (cnts==0) { + // if no user-cared-events is up + // we check to see if time is up + if (timeout!=-1) { + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + to = do_timespec_diff(&now, &abstime); + if (to.tv_sec==0 && to.tv_nsec==0) break; + } + // time is not up yet, continue loop + } + } while (cnts==0); + if (cnts>0) { + D("kevent64 epfd[%d] waiting done with [%d] events", epfd, cnts); + } + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + E("epfd[%d] epoll_wait failed", epfd); + return -1; + } + + // tell user how many events are valid + return cnts; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to) { + struct timespec delta; + delta.tv_sec = to->tv_sec - from->tv_sec; + delta.tv_nsec = to->tv_nsec - from->tv_nsec; + // norm and round up + while (delta.tv_nsec<0) { + delta.tv_sec -= 1; + delta.tv_nsec += 1000000000; + } + if (delta.tv_sec < 0) { + delta.tv_sec = 0; + delta.tv_nsec = 0; + } + return delta; +} + +int epoll_close(int epfd) { + D("epoll_closing epfd: [%d]", epfd); + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + // panic if it would be double-closed + A(eok->stopping==0, "internal logic error"); + eok->stopping = 1; + // panic if epoll_wait is pending + A(eok->waiting==0, "internal logic error"); + + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + eoks_free(eok); + + return 0; +} + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents) { + if (maxevents<=eok->nevslist) return eok->kevslist; + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kevslist, sizeof(*p)*maxevents); + if (!p) return NULL; + eok->kevslist = p; + eok->nevslist = maxevents; + return p; +} + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd) { + eok_event_t *p = eok->evs_head; + while (p) { + if (p->fd == fd) return p; + p = p->next; + } + errno = ENOENT; + return NULL; +} + +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { + eok_event_t *p = NULL; + if (eok->evs_free) { + p = eok->evs_free; + eok->evs_free = p->next; + p->next = NULL; + A(p->prev==NULL, "internal logic error"); + } else { + p = (eok_event_t*)calloc(1, sizeof(*p)); + if (!p) return NULL; + A(p->next==NULL, "internal logic error"); + A(p->prev==NULL, "internal logic error"); + } + // dirty link + p->prev = eok->evs_tail; + if (eok->evs_tail) eok->evs_tail->next = p; + else eok->evs_head = p; + eok->evs_tail = p; + + eok->evs_count += 1; + + return p; +} + +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { + if (ev->prev) ev->prev->next = ev->next; + else eok->evs_head = ev->next; + if (ev->next) ev->next->prev = ev->prev; + else eok->evs_tail = ev->prev; + ev->prev = NULL; + ev->next = eok->evs_free; + eok->evs_free = ev; + + eok->evs_count -= 1; +} + +static void eok_wakeup(ep_over_kq_t *eok) { + if (!eok->waiting) return; + if (eok->wakenup) return; + eok->wakenup = 1; + A(1==send(eok->sv[1], "1", 1, 0), ""); +} + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev) { + if (!oev) oev = eok_calloc_ev(eok); + if (!oev) return -1; + int n = 0; + if (krev->ident==ev->fd) ++n; + if (kwev->ident==ev->fd) ++n; + A(n, "internal logic error"); + if (eok->ichanges+n>eok->nchanges) { + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kchanges, sizeof(*p) * (eok->nchanges + 10)); + if (!p) { + if (ev->changed==1) { + // roll back + A(oev, "internal logic error"); + eok_free_ev(eok, oev); + } + errno = ENOMEM; + return -1; + } + eok->kchanges = p; + eok->nchanges += 10; + } + + // copy to registered event slot + oev->fd = ev->fd; + if (ev->changed!=3) { + // if add/mod, copy epoll_event + oev->epev = ev->epev; + } + oev->changed = ev->changed; + + // copy to changes list + n = 0; + if (krev->ident==ev->fd) { + krev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *krev; + ++n; + } + if (kwev->ident==ev->fd) { + kwev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *kwev; + ++n; + } + D("epfd[%d]: add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", eok->idx, n, ev->fd, eok->ichanges, eok->evs_count); + return 0; +} + +static ep_over_kq_t* eoks_alloc(void) { + ep_over_kq_t *eok = NULL; + + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (eoks.eoks_free_list) { + eok = eoks.eoks_free_list; + eoks.eoks_free_list = eok->next; + A(eoks.eoks, "internal logic error"); + A(eok->idx>=0 && eok->idxidx)==NULL, "internal logic error"); + *(eoks.eoks + eok->idx) = eok; + eok->next = NULL; + eok->stopping = 0; + break; + } + eok = (ep_over_kq_t*)calloc(1, sizeof(*eok)); + if (!eok) break; + eok->idx = -1; + ep_over_kq_t **ar = (ep_over_kq_t**)realloc(eoks.eoks, sizeof(**ar) * (eoks.neoks+1)); + if (!ar) break; + eoks.eoks = ar; + *(eoks.eoks + eoks.neoks) = eok; + eok->idx = eoks.neoks; + eok->kq = -1; + eok->sv[0] = -1; + eok->sv[1] = -1; + eoks.neoks += 1; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + + if (!eok) { + errno = ENOMEM; + return NULL; + } + if (eok->idx==-1) { + free(eok); + errno = ENOMEM; + return NULL; + } + if (eok->lock_valid) { + return eok; + } + + if (0==pthread_mutex_init(&eok->lock, NULL)) { + eok->lock_valid = 1; + return eok; + } + + eoks_free(eok); + errno = ENOMEM; + return NULL; +} + +static void eoks_free(ep_over_kq_t *eok) { + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); + + // leave eok->kchanges as is + A(eok->ichanges==0, "internal logic error"); + + A(eok->waiting==0, "internal logic error"); + eok_event_t *ev = eok->evs_head; + int sv_closed = 0; + while (ev) { + eok_event_t *next = ev->next; + if (ev->fd==eok->sv[0]) { + // fd is critical system resource + A(sv_closed==0, "internal logic error"); + close(eok->sv[0]); + eok->sv[0] = -1; + close(eok->sv[1]); + eok->sv[1] = -1; + eok_free_ev(eok, ev); + } else { + // user forget calling epoll_ctl(EPOLL_CTL_DEL) before calling epoll_close/close? + // calling close(ev->fd) here smells really bad +#ifndef BALANCE_CHECK_WHEN_CLOSE + // we just let it be and reclaim ev + eok_free_ev(eok, ev); +#else + // panic otherwise, if BALANCE_CHECK_WHEN_CLOSE is defined + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "epfd[%d] fd[%d]: internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?", + eok->idx, ev->fd); +#endif + } + ev = next; + } + // if (eok->evs_count==1) { + // A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); + // A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); + // // fd is critical system resource + // close(eok->sv[0]); + // eok->sv[0] = -1; + // close(eok->sv[1]); + // eok->sv[1] = -1; + // eok_free_ev(eok, eok->evs_head); + // } + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?"); + A(eok->sv[0]==-1 && eok->sv[1]==-1, "internal logic error"); + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + eok->next = eoks.eoks_free_list; + eoks.eoks_free_list = eok; + *(eoks.eoks + eok->idx) = NULL; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); +} + +static ep_over_kq_t* eoks_find(int epfd) { + ep_over_kq_t *eok = NULL; + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (epfd<0 || epfd>=eoks.neoks) { + break; + } + A(eoks.eoks, "internal logic error"); + eok = *(eoks.eoks + epfd); + A(eok->next==NULL, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + return eok; +} + diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index 0d5c130d6eddce07b967e3fe15762f01d6e1e97b..facfbd23af7a579ed11655ce66dddd971677fb18 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -13,3 +13,4 @@ TARGET_LINK_LIBRARIES(osdetail os) IF (TD_ARM_32 OR TD_LINUX_32) TARGET_LINK_LIBRARIES(osdetail atomic) ENDIF () + diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index d3f0fda1a5c8535226eedbac6d5d88a1352e3c6e..144c59fa2db3ae24a81679de16116812fddbccc5 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -51,16 +51,22 @@ int taosMkDir(const char *path, mode_t mode) { return code; } -void taosRename(char* oldName, char *newName) { - // if newName in not empty, rename return fail. - // the newName must be empty or does not exist - if (rename(oldName, newName)) { + +#ifndef TAOS_OS_FUNC_DIR + +int32_t taosRename(char* oldName, char *newName) { + int32_t code = rename(oldName, newName); + if (code < 0) { uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } else { - uInfo("successfully to rename file %s to %s", oldName, newName); + uTrace("successfully to rename file %s to %s", oldName, newName); } + + return code; } +#endif + void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { DIR *dir = opendir(rootDir); if (dir == NULL) return; @@ -117,7 +123,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { goto cmp_end; } - int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { ret = -2; goto cmp_end; diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 2e6886aa21f8c083e1134122ac334a20edf54874..538ed378798f53c09c8cb8ada2f2a1a066d8b6bf 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -25,7 +25,8 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { char tmpPath[PATH_MAX]; int32_t len = strlen(tsTempDir); memcpy(tmpPath, tsTempDir, len); - + static uint64_t seqId = 0; + if (tmpPath[len - 1] != '/') { tmpPath[len++] = '/'; } @@ -36,8 +37,10 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, "-%d-%s"); } - char rand[8] = {0}; - taosRandStr(rand, tListLen(rand) - 1); + char rand[32] = {0}; + + sprintf(rand, "%"PRIu64, atomic_add_fetch_64(&seqId, 1)); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); } @@ -119,17 +122,49 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int64_t taosCopy(char *from, char *to) { + char buffer[4096]; + int fidto = -1, fidfrom = -1; + int64_t size = 0; + int64_t bytes; + + fidfrom = open(from, O_RDONLY | O_BINARY); + if (fidfrom < 0) goto _err; + + fidto = open(to, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755); + if (fidto < 0) goto _err; + + while (true) { + bytes = taosRead(fidfrom, buffer, sizeof(buffer)); + if (bytes < 0) goto _err; + if (bytes == 0) break; + + size += bytes; + + if (taosWrite(fidto, (void *)buffer, bytes) < bytes) goto _err; + if (bytes < sizeof(buffer)) break; + } + + fsync(fidto); + + close(fidfrom); + close(fidto); + return size; + +_err: + if (fidfrom >= 0) close(fidfrom); + if (fidto >= 0) close(fidto); + remove(to); + return -1; +} + #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; while (leftbytes > 0) { - /* - * TODO : Think to check if file is larger than 1GB - */ - // if (leftbytes > 1000000000) leftbytes = 1000000000; sentbytes = sendfile(dfd, sfd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index 53310d179c0090382e009de949e5158146dc282a..291a54b6695106ba3b457d148b1439e283d6ceff 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -512,8 +512,9 @@ void * taosTRealloc(void *ptr, size_t size) { return (void *)((char *)tptr + sizeof(size_t)); } -void taosTZfree(void *ptr) { +void* taosTZfree(void* ptr) { if (ptr) { - free((void *)((char *)ptr - sizeof(size_t))); + free((void*)((char*)ptr - sizeof(size_t))); } + return NULL; } \ No newline at end of file diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 9eb8c18a40a11fac9ada037163011cdcf92201fb..2bf2f2448752bf468497fc48a8871c5f0120b9ab 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -31,7 +31,15 @@ int tsem_wait(tsem_t* sem) { #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } -int64_t taosGetPthreadId() { return (int64_t)pthread_self(); } + +int64_t taosGetSelfPthreadId() { + static __thread int id = 0; + if (id != 0) return id; + id = syscall(SYS_gettid); + return id; +} + +int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } void taosResetPthread(pthread_t *thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } int32_t taosGetPId() { return getpid(); } @@ -61,4 +69,4 @@ int32_t taosGetCurrentAPPName(char *name, int32_t* len) { return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c new file mode 100644 index 0000000000000000000000000000000000000000..e1a0e84e7fe689bdf92a9a7bafbddb80ccd03a4b --- /dev/null +++ b/src/os/src/detail/osSignal.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tulog.h" + +#ifndef TAOS_OS_FUNC_SIGNAL +typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + struct sigaction act = {{0}}; +#if 1 + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = (FLinuxSignalHandler)sigfp; +#else + act.sa_handler = sigfp; +#endif + sigaction(signum, &act, NULL); +} + +void taosIgnSignal(int32_t signum) { + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + signal(signum, SIG_DFL); +} + +#endif diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index c7c9d774271555ae5989aa18b7e00d325a0eddde..1186a6dd0a0247f3df44f1065fdacfa7d8df6e07 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -39,6 +39,10 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() { + signal(SIGPIPE, SIG_IGN); +} + void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); @@ -49,6 +53,16 @@ void taosBlockSIGPIPE() { } } +void taosSetMaskSIGPIPE() { + sigset_t signal_mask; + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGPIPE); + int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); + if (rc != 0) { + uError("failed to setmask SIGPIPE"); + } +} + #endif #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT @@ -69,4 +83,4 @@ const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/detail/osString.c b/src/os/src/detail/osString.c index 31ffd6b87cf3a63b8663c87514699f5579ffc061..59f47b4815badd8411b54ea997bf6b46f3a18a27 100644 --- a/src/os/src/detail/osString.c +++ b/src/os/src/detail/osString.c @@ -46,7 +46,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { return (int32_t)(ucs4_max_len - outLen); } -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { memset(ucs4, 0, ucs4_max_len); iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); size_t ucs4_input_len = mbsLength; @@ -58,7 +58,7 @@ bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len iconv_close(cd); if (len != NULL) { - *len = ucs4_max_len - outLeft; + *len = (int32_t)(ucs4_max_len - outLeft); } return true; @@ -92,7 +92,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { return len; } -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { memset(ucs4, 0, ucs4_max_len); mbstate_t state = {0}; int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index b0ca6139ed9461b12ab2845d0060b51dc388bcab..f12ec93bf7725d013fa2f53d148de783e854d3bf 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -18,6 +18,7 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" +#include "taoserror.h" #ifndef TAOS_OS_FUNC_SYSINFO @@ -45,6 +46,21 @@ static char tsProcMemFile[25] = {0}; static char tsProcIOFile[25] = {0}; static float tsPageSizeKB = 0; +static void taosGetProcInfos() { + tsPageSize = sysconf(_SC_PAGESIZE); + tsOpenMax = sysconf(_SC_OPEN_MAX); + tsStreamMax = sysconf(_SC_STREAM_MAX); + + tsProcId = (pid_t)syscall(SYS_gettid); + tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; + + snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); + snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); + snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); +} + +static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } + bool taosGetSysMemory(float *memoryUsedMB) { float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; @@ -105,7 +121,8 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { } char cpu[10] = {0}; - sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); + sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, + &cpuInfo->idle); tfree(line); fclose(fp); @@ -131,7 +148,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { for (int i = 0, blank = 0; line[i] != 0; ++i) { if (line[i] == ' ') blank++; if (blank == PROCESS_ITEM) { - sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); + sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, + &cpuInfo->cutime, &cpuInfo->cstime); break; } } @@ -162,12 +180,12 @@ static void taosGetSystemTimezone() { char buf[68] = {0}; if (f != NULL) { int len = fread(buf, 64, 1, f); - if(len < 64 && ferror(f)) { + if (len < 64 && ferror(f)) { fclose(f); uError("read /etc/timezone error, reason:%s", strerror(errno)); return; } - + fclose(f); buf[sizeof(buf) - 1] = 0; @@ -231,7 +249,7 @@ static void taosGetSystemLocale() { // get and set default locale if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { locale = setlocale(LC_CTYPE, ""); if (locale == NULL) { - uError("can't get locale from system, set it to en_US.UTF-8"); + uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); strcpy(tsLocale, "en_US.UTF-8"); } else { tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); @@ -258,6 +276,8 @@ static void taosGetSystemLocale() { // get and set default locale } } +static int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } + bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { static uint64_t lastSysUsed = 0; static uint64_t lastSysTotal = 0; @@ -297,43 +317,17 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { return true; } -bool taosGetDisk() { +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { struct statvfs info; - const double unit = 1024 * 1024 * 1024; - - if (tscEmbedded) { - if (statvfs(tsDataDir, &info)) { - //tsTotalDataDirGB = 0; - //tsAvailDataDirGB = 0; - uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - return false; - } else { - tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); - tsAvailDataDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit); - } - } - - if (statvfs(tsLogDir, &info)) { - //tsTotalLogDirGB = 0; - //tsAvailLogDirGB = 0; - uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno)); - return false; - } else { - tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); - tsAvailLogDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit); - } - - if (statvfs("/tmp", &info)) { - //tsTotalTmpDirGB = 0; - //tsAvailTmpDirectorySpace = 0; - uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno)); - return false; + if (statvfs(tsDataDir, &info)) { + uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } else { - tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); - tsAvailTmpDirectorySpace = (float)((double)info.f_bavail * (double)info.f_frsize / unit); + diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->avail = info.f_bavail * info.f_frsize; + return 0; } - - return true; } static bool taosGetCardInfo(int64_t *bytes) { @@ -344,13 +338,12 @@ static bool taosGetCardInfo(int64_t *bytes) { return false; } - size_t len = 2048; char * line = calloc(1, len); while (!feof(fp)) { memset(line, 0, len); - + int64_t rbytes = 0; int64_t rpackts = 0; int64_t tbytes = 0; @@ -465,7 +458,7 @@ bool taosGetProcIO(float *readKB, float *writeKB) { static int64_t lastReadbyte = -1; static int64_t lastWritebyte = -1; - int64_t curReadbyte = 0; + int64_t curReadbyte = 0; int64_t curWritebyte = 0; if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { @@ -490,23 +483,15 @@ bool taosGetProcIO(float *readKB, float *writeKB) { } void taosGetSystemInfo() { - tsNumOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - tsPageSize = sysconf(_SC_PAGESIZE); - tsOpenMax = sysconf(_SC_OPEN_MAX); - tsStreamMax = sysconf(_SC_STREAM_MAX); + taosGetProcInfos(); - tsProcId = (pid_t)syscall(SYS_gettid); - tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; - tsTotalMemoryMB = (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); - - snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); - snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); - snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); float tmp1, tmp2; taosGetSysMemory(&tmp1); taosGetProcMemory(&tmp2); - taosGetDisk(); + // taosGetDisk(); taosGetBandSpeed(&tmp1); taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); @@ -533,7 +518,6 @@ void taosPrintOsInfo() { uInfo(" os release: %s", buf.release); uInfo(" os version: %s", buf.version); uInfo(" os machine: %s", buf.machine); - uInfo("=================================="); } void taosKillSystem() { @@ -573,16 +557,16 @@ void taosSetCoreDump() { if (0 == tsEnableCoreFile) { return; } - + // 1. set ulimit -c unlimited struct rlimit rlim; struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { @@ -594,57 +578,56 @@ void taosSetCoreDump() { } if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif } #ifndef _TD_ARM_ // 2. set the path for saving core file struct __sysctl_args args; - int old_usespid = 0; - size_t old_len = 0; - int new_usespid = 1; - size_t new_len = sizeof(new_usespid); - + + int old_usespid = 0; + size_t old_len = 0; + int new_usespid = 1; + size_t new_len = sizeof(new_usespid); + int name[] = {CTL_KERN, KERN_CORE_USES_PID}; - + memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - args.newval = &new_usespid; - args.newlen = new_len; - + args.newval = &new_usespid; + args.newlen = new_len; + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } - - uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); old_usespid = 0; - old_len = 0; + old_len = 0; memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } - + uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif - } bool taosGetSystemUid(char *uid) { diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index b78627f46f2124912338f3b029b5a0fd341e729c..d9d070218e2a64e5f2535c073db4852b0a8960ef 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -43,7 +43,7 @@ */ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, const unsigned int day, const unsigned int hour, - const unsigned int min, const unsigned int sec) + const unsigned int min, const unsigned int sec, int64_t timezone) { unsigned int mon = mon0, year = year0; @@ -61,12 +61,6 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, res = res*24; res = ((res + hour) * 60 + min) * 60 + sec; -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t timezone = _timezone; -#endif -#endif - return (res + timezone); } @@ -219,7 +213,7 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { /* mktime will be affected by TZ, set by using taos_options */ #ifdef WINDOWS - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); //int64_t seconds = gmtime(&tm); #else int64_t seconds = timegm(&tm); @@ -276,7 +270,13 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { return -1; } - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + int64_t timezone = _timezone; +#endif +#endif + + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); int64_t fraction = 0; @@ -486,7 +486,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (delta / pInterval->sliding + factor) * pInterval->sliding; if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* + /* * here we revised the start time of day according to the local time zone, * but in case of DST, the start time of one day need to be dynamically decided. */ @@ -501,9 +501,24 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); } - int64_t end = start + pInterval->interval - 1; - if (end < t) { - start += pInterval->sliding; + int64_t end = 0; + + // not enough time range + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + + while(end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window + start += pInterval->sliding; + + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + } else { + end = INT64_MAX; + break; + } + } + } else { + end = INT64_MAX; } } @@ -559,4 +574,4 @@ const char* fmtts(int64_t ts) { } return buf; -} \ No newline at end of file +} diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index 9883a03a0933075616b69800fe6e34a36fc6c746..c1135ce292c4ef0a5ae968525ca2685a97b6f62e 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -89,12 +89,17 @@ int taosInitTimer(void (*callback)(int), int ms) { if (code != 0) { uError("failed to create timer thread"); return -1; + } else { + uDebug("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); } + return 0; } void taosUninitTimer() { stopTimer = true; + + uDebug("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); pthread_join(timerThread, NULL); } @@ -111,6 +116,9 @@ void taosUninitTimer() { pthread_sigmask(SIG_BLOCK, &set, NULL); */ void taosMsleep(int mseconds) { +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ #if 1 usleep(mseconds * 1000); #else @@ -131,6 +139,7 @@ void taosMsleep(int mseconds) { /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ #endif + } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 14b40a1f1857ca7789829cb64f596ff349b7c997..e3eadbc94bcf71e7cf09e70901e4de0aed427c54 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -18,7 +18,6 @@ #include "tglobal.h" void osInit() { - #ifdef _TD_POWER_ if (configDir[0] == 0) { strcpy(configDir, "/etc/power"); @@ -39,4 +38,20 @@ void osInit() { strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); strcpy(tsOsName, "Linux"); -} \ No newline at end of file +} + +char cmdline[1024]; + +char* taosGetCmdlineByPID(int pid) { + sprintf(cmdline, "/proc/%d/cmdline", pid); + FILE* f = fopen(cmdline, "r"); + if (f) { + size_t size; + size = fread(cmdline, sizeof(char), 1024, f); + if (size > 0) { + if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0'; + } + fclose(f); + } + return cmdline; +} diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 2204135ae657eb813144dc2bd1ae637d9510842e..3bfe1c1b5dd24185407b2e2cd1b94ff621089b0b 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -15,18 +15,19 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "osSocket.h" #include "tglobal.h" +#include "tulog.h" void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + const char *tdengineTmpFileNamePrefix = "tdengine-"; + char tmpPath[PATH_MAX]; int32_t len = (int32_t)strlen(tsTempDir); memcpy(tmpPath, tsTempDir, len); if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') { - tmpPath[len++] = '\\'; + tmpPath[len++] = '\\'; } strcpy(tmpPath + len, tdengineTmpFileNamePrefix); @@ -35,7 +36,7 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); } - + char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); @@ -46,18 +47,16 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { + } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); return (int64_t)(writeLen + rlen); - } - else { + } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); writeLen += _SEND_FILE_STEP_; } @@ -65,12 +64,44 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co int64_t remain = count - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, (size_t) remain, in_file); + size_t rlen = fread(buffer, 1, (size_t)remain, in_file); + if (rlen <= 0) { + return writeLen; + } else { + fwrite(buffer, 1, (size_t)remain, out_file); + writeLen += remain; + } + } + + return writeLen; +} + +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { + if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); + + int64_t writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); if (rlen <= 0) { return writeLen; + } else if (rlen < _SEND_FILE_STEP_) { + taosWriteSocket(dfd, buffer, rlen); + return (int64_t)(writeLen + rlen); + } else { + taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); + writeLen += _SEND_FILE_STEP_; } - else { - fwrite(buffer, 1, (size_t) remain, out_file); + } + + int64_t remain = count - writeLen; + if (remain > 0) { + int32_t rlen = read(sfd, buffer, (int32_t)remain); + if (rlen <= 0) { + return writeLen; + } else { + taosWriteSocket(sfd, buffer, (int32_t)remain); writeLen += remain; } } @@ -78,12 +109,76 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { - uError("taosSendFile no implemented yet"); +int32_t taosFtruncate(int32_t fd, int64_t l_size) { + if (fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + return 0; } -int32_t taosFtruncate(int32_t fd, int64_t length) { - uError("taosFtruncate no implemented yet"); - return 0; +int fsync(int filedes) { + if (filedes < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(filedes); + + return FlushFileBuffers(h); +} + +int32_t taosRename(char* oldName, char *newName) { + int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (code < 0) { + uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } else { + uTrace("successfully to rename file %s to %s", oldName, newName); + } + + return code; } \ No newline at end of file diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemphone.c index 1f723540f695e4a4609a9ec74e773a1559e7f30a..a3f0367ee1179f74769c8883afe5ee47237ac32b 100644 --- a/src/os/src/windows/wSemphone.c +++ b/src/os/src/windows/wSemphone.c @@ -20,19 +20,24 @@ #include "ttimer.h" #include "tulog.h" #include "tutil.h" +#include bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } void taosResetPthread(pthread_t *thread) { thread->p = 0; } -int64_t taosGetPthreadId() { +int64_t taosGetPthreadId(pthread_t thread) { #ifdef PTW32_VERSION - return pthread_getw32threadid_np(pthread_self()); + return pthread_getw32threadid_np(thread); #else - return (int64_t)pthread_self(); + return (int64_t)thread; #endif } +int64_t taosGetSelfPthreadId() { + return GetCurrentThreadId(); +} + bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } @@ -53,4 +58,4 @@ int32_t taosGetCurrentAPPName(char *name, int32_t* len) { } return 0; -} \ No newline at end of file +} diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c new file mode 100644 index 0000000000000000000000000000000000000000..9de6b5f3434dc792c40b330ec37c7edd180b999e --- /dev/null +++ b/src/os/src/windows/wSignal.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include +#include + +typedef void (*FWinSignalHandler)(int32_t signum); + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + if (signum == SIGUSR1) return; + + // SIGHUP doesn't exist in windows, we handle it in the way of ctrlhandler + if (signum == SIGHUP) { + SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); + } else { + signal(signum, (FWinSignalHandler)sigfp); + } +} + +void taosIgnSignal(int32_t signum) { + if (signum == SIGUSR1 || signum == SIGHUP) return; + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + if (signum == SIGUSR1 || signum == SIGHUP) return; + signal(signum, SIG_DFL); +} diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 3b091b269931e644d9f8c2c01ba3c9cb9ddc520c..4e6ee15e772c5af56b4ebd11e74fc44dd57946f5 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -46,7 +46,9 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} +void taosSetMaskSIGPIPE() {} int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { diff --git a/src/os/src/windows/wString.c b/src/os/src/windows/wString.c index 1fb235a00575529be660cea18c1401f1d075a49b..67237e655cd2916ec2ec80daa01e7a0f9ae0d3e3 100644 --- a/src/os/src/windows/wString.c +++ b/src/os/src/windows/wString.c @@ -75,18 +75,6 @@ char *getpass(const char *prefix) { return passwd; } -char *strndup(const char *s, size_t n) { - size_t len = strlen(s); - if (len >= n) { - len = n; - } - - char *r = calloc(len + 1, 1); - memcpy(r, s, len); - r[len] = 0; - return r; -} - int twcslen(const wchar_t *wcs) { int *wstr = (int *)wcs; if (NULL == wstr) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java b/src/os/src/windows/wSysLog.c similarity index 69% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java rename to src/os/src/windows/wSysLog.c index c1b9b02fa85f8e2197f41d75f8f0c25b3f4c416c..866cacbabaa870a6fbe58f1a3dc363dbfdd80949 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java +++ b/src/os/src/windows/wSysLog.c @@ -1,4 +1,4 @@ -/*************************************************************************** +/* * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify @@ -11,9 +11,9 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; + */ -public interface TSDBSubscribeCallBack { - void invoke(TSDBResultSet resultSet); -} +#define _DEFAULT_SOURCE +#include "os.h" + +void syslog(int unused, const char *format, ...) {} \ No newline at end of file diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 1bfee25c4a6a1851bdfbc16e795185412f007df2..48fb3c13a8fe80332cfc26f893d6a21f1cdd902e 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -21,6 +21,7 @@ #include "ttimer.h" #include "tulog.h" #include "tutil.h" +#include "taoserror.h" #if (_WIN64) #include #include @@ -31,11 +32,54 @@ #pragma comment(lib, "Mswsock.lib ") #endif +#include + #pragma warning(push) -#pragma warning(disable:4091) +#pragma warning(disable : 4091) #include #pragma warning(pop) +static int32_t taosGetTotalMemory() { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return 0; + } + + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + return (int32_t)nMemTotal; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return false; + } + + float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + + *memoryUsedMB = nMemTotal - nMemFree; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + unsigned bytes_used = 0; + +#if defined(_WIN64) && defined(_MSC_VER) + PROCESS_MEMORY_COUNTERS pmc; + HANDLE cur_proc = GetCurrentProcess(); + + if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { + bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); + } +#endif + + *memoryUsedMB = (float)bytes_used / 1024 / 1024; + return true; +} + static void taosGetSystemTimezone() { // get and set default timezone SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); @@ -71,38 +115,38 @@ static void taosGetSystemLocale() { } } -void taosPrintOsInfo() {} - -void taosKillSystem() { - uError("function taosKillSystem, exit!"); - exit(0); +static int32_t taosGetCpuCores() { + SYSTEM_INFO info; + GetSystemInfo(&info); + return (int32_t)info.dwNumberOfProcessors; } -void taosGetSystemInfo() { - taosGetSystemTimezone(); - taosGetSystemLocale(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; } -bool taosGetDisk() { - const double unit = 1024 * 1024 * 1024; - BOOL fResult; +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { unsigned _int64 i64FreeBytesToCaller; unsigned _int64 i64TotalBytes; unsigned _int64 i64FreeBytes; - char dir[4] = {'C', ':', '\\', '\0'}; - int drive_type; - - if (tscEmbedded) { - drive_type = GetDriveTypeA(dir); - if (drive_type == DRIVE_FIXED) { - fResult = GetDiskFreeSpaceExA(dir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, - (PULARGE_INTEGER)&i64FreeBytes); - if (fResult) { - tsTotalDataDirGB = tsTotalLogDirGB = tsTotalTmpDirGB = (float)(i64TotalBytes / unit); - tsAvailDataDirGB = tsAvailLogDirGB = tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit); - } - } + + BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + diskSize->tsize = (int64_t)(i64TotalBytes); + diskSize->avail = (int64_t)(i64FreeBytes); + return 0; + } else { + uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; return true; } @@ -144,48 +188,30 @@ bool taosGetProcIO(float *readKB, float *writeKB) { return true; } -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; -} - -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return true; -} - -bool taosGetProcMemory(float *memoryUsedMB) { - unsigned bytes_used = 0; -#if 0 -#if defined(_WIN32) && defined(_MSC_VER) - PROCESS_MEMORY_COUNTERS pmc; - HANDLE cur_proc = GetCurrentProcess(); - - if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { - bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); - } -#endif -#endif +void taosGetSystemInfo() { + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); - *memoryUsedMB = (float)bytes_used / 1024 / 1024; + float tmp1, tmp2; + // taosGetDisk(); + taosGetBandSpeed(&tmp1); + taosGetCpuUsage(&tmp1, &tmp2); + taosGetProcIO(&tmp1, &tmp2); - return true; + taosGetSystemTimezone(); + taosGetSystemLocale(); } -bool taosGetSysMemory(float *memoryUsedMB) { - MEMORYSTATUSEX memsStat; - float nMemFree; - float nMemTotal; +void taosPrintOsInfo() { + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + uInfo("=================================="); +} - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return false; - } - nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); - nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - *memoryUsedMB = nMemTotal - nMemFree; - return true; +void taosKillSystem() { + uError("function taosKillSystem, exit!"); + exit(0); } int taosSystem(const char *cmd) { @@ -195,10 +221,6 @@ int taosSystem(const char *cmd) { int flock(int fd, int option) { return 0; } -int fsync(int filedes) { return 0; } - -int sigaction(int sig, struct sigaction *d, void *p) { return 0; } - LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile, IN MINIDUMP_TYPE DumpType, @@ -235,4 +257,22 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { return EXCEPTION_CONTINUE_SEARCH; } -void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } \ No newline at end of file +void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } + +bool taosGetSystemUid(char *uid) { + GUID guid; + CoCreateGuid(&guid); + + sprintf( + uid, + "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", + guid.Data1, guid.Data2, guid.Data3, + guid.Data4[0], guid.Data4[1], + guid.Data4[2], guid.Data4[3], + guid.Data4[4], guid.Data4[5], + guid.Data4[6], guid.Data4[7]); + + return true; +} + +char *taosGetCmdlineByPID(int pid) { return ""; } diff --git a/src/os/src/windows/wWordexp.c b/src/os/src/windows/wWordexp.c index bb9acde25a5de6c334f512fc8994edffdf5c59a3..febe22ac8fa8566607fe2bf9b0be9cc7443c8012 100644 --- a/src/os/src/windows/wWordexp.c +++ b/src/os/src/windows/wWordexp.c @@ -21,13 +21,20 @@ #include "tulog.h" #include "tutil.h" -int wordexp(const char *words, wordexp_t *pwordexp, int flags) { +int wordexp(char *words, wordexp_t *pwordexp, int flags) { pwordexp->we_offs = 0; pwordexp->we_wordc = 1; - pwordexp->we_wordv = (char **)(pwordexp->wordPos); - pwordexp->we_wordv[0] = (char *)words; + pwordexp->we_wordv[0] = pwordexp->wordPos; + + memset(pwordexp->wordPos, 0, 1025); + if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { + pwordexp->we_wordv[0] = words; + uError("failed to parse relative path:%s to abs path", words); + return -1; + } + + uTrace("parse relative path:%s to abs path:%s", words, pwordexp->wordPos); return 0; } void wordfree(wordexp_t *pwordexp) {} - diff --git a/src/os/tests/CMakeLists.txt b/src/os/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1a18a72b40677f1e21b2431499b17c4c745fad7d --- /dev/null +++ b/src/os/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(osTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(osTest taos osdetail tutil common gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/os/tests/test.cpp b/src/os/tests/test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..600e5d71a760653f8a932985f0d14a60081575b6 --- /dev/null +++ b/src/os/tests/test.cpp @@ -0,0 +1,34 @@ +#include "os.h" +#include +#include +#include + +#include "taos.h" +#include "tstoken.h" +#include "tutil.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +// test function in os module +TEST(testCase, parse_time) { + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + deltaToUtcInitOnce(); + + // window: 1500000001000, 1500002000000 + // pQuery->interval: interval: 86400000, sliding:3600000 + int64_t key = 1500000001000; + SInterval interval = {0}; + interval.interval = 86400000; + interval.intervalUnit = 'd'; + interval.sliding = 3600000; + interval.slidingUnit = 'h'; + + int64_t s = taosTimeTruncate(key, &interval, TSDB_TIME_PRECISION_MILLI); + ASSERT_TRUE(s + interval.interval >= key); +} + + + diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 742235894a8781e4dcfa09cb86985c73df74d046..7dcaaf27e615ead75e83630788288a27e938b0a9 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -3,6 +3,6 @@ PROJECT(TDengine) ADD_SUBDIRECTORY(monitor) ADD_SUBDIRECTORY(http) -IF (TD_MQTT) +IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(mqtt) ENDIF () \ No newline at end of file diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 5d8b52986a6bb1cc451dac10cd92c8c37a352c4b..bfb47ad12e8b1ef7099109ecf5849ec3575caf5f 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -8,18 +8,16 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX) - ADD_LIBRARY(http ${SRC}) - TARGET_LINK_LIBRARIES(http z) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(http taos_static) - ELSE () - TARGET_LINK_LIBRARIES(http taos) - ENDIF () +ADD_LIBRARY(http ${SRC}) +TARGET_LINK_LIBRARIES(http z) - IF (TD_ADMIN) - TARGET_LINK_LIBRARIES(http admin) - ENDIF () +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(http taos_static) +ELSE () + TARGET_LINK_LIBRARIES(http taos) +ENDIF () + +IF (TD_ADMIN) + TARGET_LINK_LIBRARIES(http admin) ENDIF () diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h index 260858c5cc22aa10dc49a61743879d768907c8fe..b016da2dd3a75bc4e51f29e3f9344458d11badea 100644 --- a/src/plugins/http/inc/httpContext.h +++ b/src/plugins/http/inc/httpContext.h @@ -22,7 +22,7 @@ bool httpInitContexts(); void httpCleanupContexts(); const char *httpContextStateStr(HttpContextState state); -HttpContext *httpCreateContext(int32_t fd); +HttpContext *httpCreateContext(SOCKET fd); bool httpInitContext(HttpContext *pContext); HttpContext *httpGetContext(void * pContext); void httpReleaseContext(HttpContext *pContext, bool clearRes); diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index ebdfabf3101001d020e24bf63c3d699492ff21d0..634468f3ccfd041300e0b99c728a8fb6a9b8fbae 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_HTTP_INT_H #define TDENGINE_HTTP_INT_H +#include "os.h" #include #include "pthread.h" #include "semaphore.h" @@ -39,7 +40,8 @@ #define HTTP_GC_TARGET_SIZE 512 #define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_WAIT_TIME_MS 5 -#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + TSDB_PASSWORD_LEN) +#define HTTP_PASSWORD_LEN TSDB_UNI_LEN +#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + HTTP_PASSWORD_LEN) typedef enum HttpReqType { HTTP_REQTYPE_OTHERS = 0, @@ -139,7 +141,7 @@ typedef enum { typedef struct HttpContext { int32_t refCount; - int32_t fd; + SOCKET fd; uint32_t accessTimes; uint32_t lastAccessTime; int32_t state; @@ -147,7 +149,7 @@ typedef struct HttpContext { uint8_t parsed; char ipstr[22]; char user[TSDB_USER_LEN]; // parsed from auth token or login message - char pass[TSDB_PASSWORD_LEN]; + char pass[HTTP_PASSWORD_LEN]; TAOS * taos; void * ppContext; HttpSession *session; @@ -166,7 +168,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - int32_t pollFd; + EpollFd pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -177,7 +179,9 @@ typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; uint32_t serverIp; uint16_t serverPort; - int32_t fd; + int8_t stop; + int8_t reserve; + SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/inc/httpRestJson.h b/src/plugins/http/inc/httpRestJson.h index 112e845f369c6ae3f0df76d880f84de5aeff5d8d..5f19983826a4f90bc3498ca82cc9fd495b4f300c 100644 --- a/src/plugins/http/inc/httpRestJson.h +++ b/src/plugins/http/inc/httpRestJson.h @@ -34,6 +34,8 @@ #define REST_JSON_DATA_LEN 4 #define REST_JSON_HEAD "head" #define REST_JSON_HEAD_LEN 4 +#define REST_JSON_HEAD_INFO "column_meta" +#define REST_JSON_HEAD_INFO_LEN 11 #define REST_JSON_ROWS "rows" #define REST_JSON_ROWS_LEN 4 #define REST_JSON_AFFECT_ROWS "affected_rows" @@ -51,4 +53,4 @@ bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAO bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd); -#endif \ No newline at end of file +#endif diff --git a/src/plugins/http/inc/httpUtil.h b/src/plugins/http/inc/httpUtil.h index 61cd50a77a29ad8b9a2058f4ad569493b18618ac..54c95b6980f8241c3ea6c8e563e0e42c7c737286 100644 --- a/src/plugins/http/inc/httpUtil.h +++ b/src/plugins/http/inc/httpUtil.h @@ -17,7 +17,7 @@ #define TDENGINE_HTTP_UTIL_H bool httpCheckUsedbSql(char *sql); -void httpTimeToString(time_t t, char *buf, int32_t buflen); +void httpTimeToString(int32_t t, char *buf, int32_t buflen); bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp); bool httpParseRequest(HttpContext *pContext); diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index 8beef7c042ba6f0194d814753e15387e3d2b4a2b..2ef5406823b63862ba8e529ed4caea5bf7e4ec32 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -26,7 +26,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; int32_t outlen = 0; - char *base64 = (char *)base64_decode(token, len, &outlen); + char * base64 = (char *)base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); free(base64); @@ -49,9 +49,9 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) strncpy(pContext->user, base64, (size_t)user_len); pContext->user[user_len] = 0; - char *password = user + 1; + char * password = user + 1; int32_t pass_len = (int32_t)((base64 + outlen) - password); - if (pass_len < 1 || pass_len >= TSDB_PASSWORD_LEN) { + if (pass_len < 1 || pass_len >= HTTP_PASSWORD_LEN) { httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); free(base64); return -1; @@ -66,14 +66,14 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; - int32_t outlen = 0; + int32_t outlen = 0; unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); if (base64) free(base64); return 01; } - if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) { + if (outlen != (TSDB_USER_LEN + HTTP_PASSWORD_LEN)) { httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token); free(base64); return -1; @@ -97,14 +97,14 @@ int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) } int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { - char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; + char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; size_t size = sizeof(pContext->user); tstrncpy(buffer, pContext->user, size); size = sizeof(pContext->pass); tstrncpy(buffer + sizeof(pContext->user), pContext->pass, size); - char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + TSDB_PASSWORD_LEN); - char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + TSDB_PASSWORD_LEN); + char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + HTTP_PASSWORD_LEN); + char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + HTTP_PASSWORD_LEN); size_t len = strlen(base64); tstrncpy(token, base64, len + 1); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 22f464924e2c21cdeccb0c152457fad11a10294a..13f706af653e9c5c1b9fb0a4c602355b001d0cac 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -35,20 +35,24 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; if (pContext->fd >= 0) { epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - int32_t fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#ifdef WINDOWS + SOCKET fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#else + SOCKET fd = atomic_val_compare_exchange_64(&pContext->fd, pContext->fd, -1); +#endif taosCloseSocket(fd); } } static void httpDestroyContext(void *data) { HttpContext *pContext = *(HttpContext **)data; - if (pContext->fd > 0) taosClose(pContext->fd); + if (pContext->fd > 0) taosCloseSocket(pContext->fd); HttpThread *pThread = pContext->pThread; httpRemoveContextFromEpoll(pContext); httpReleaseSession(pContext); atomic_sub_fetch_32(&pThread->numOfContexts, 1); - + httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount, data, pContext->pThread->label, pContext->pThread->numOfContexts); pContext->pThread = 0; @@ -79,7 +83,7 @@ bool httpInitContexts() { void httpCleanupContexts() { if (tsHttpServer.contextCache != NULL) { SCacheObj *cache = tsHttpServer.contextCache; - httpInfo("context cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable)); + httpInfo("context cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable)); taosCacheCleanup(tsHttpServer.contextCache); tsHttpServer.contextCache = NULL; } @@ -100,15 +104,13 @@ const char *httpContextStateStr(HttpContextState state) { } } -void httpNotifyContextClose(HttpContext *pContext) { - shutdown(pContext->fd, SHUT_WR); -} +void httpNotifyContextClose(HttpContext *pContext) { shutdown(pContext->fd, SHUT_WR); } bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) { return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); } -HttpContext *httpCreateContext(int32_t fd) { +HttpContext *httpCreateContext(SOCKET fd) { HttpContext *pContext = calloc(1, sizeof(HttpContext)); if (pContext == NULL) return NULL; @@ -123,8 +125,8 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); - // set the ref to 0 - taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false); + // set the ref to 0 + taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false); return pContext; } @@ -152,7 +154,10 @@ void httpReleaseContext(HttpContext *pContext, bool clearRes) { } if (clearRes) { - httpClearParser(pContext->parser); + if (pContext->parser) { + httpClearParser(pContext->parser); + } + memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); } HttpContext **ppContext = pContext->ppContext; @@ -174,7 +179,6 @@ bool httpInitContext(HttpContext *pContext) { pContext->encodeMethod = NULL; memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); return true; } @@ -184,9 +188,9 @@ void httpCloseContextByApp(HttpContext *pContext) { pContext->parsed = false; bool keepAlive = true; - if (parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) { + if (parser && parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) { keepAlive = false; - } else if (parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) { + } else if (parser && parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) { keepAlive = false; } else { } diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index 5d4cb0c6809d063107cb37ccaac1200f2c87ee9d..925c74e7cd46751ada1e04873710a329e77bded0 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -59,11 +59,11 @@ bool gcGetUserFromUrl(HttpContext* pContext) { bool gcGetPassFromUrl(HttpContext* pContext) { HttpParser* pParser = pContext->parser; - if (pParser->path[GC_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) { + if (pParser->path[GC_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, TSDB_PASSWORD_LEN); + tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, HTTP_PASSWORD_LEN); return true; } @@ -192,7 +192,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { break; } - cJSON* alias = cJSON_GetObjectItem(query, "alias"); + cJSON* alias = cJSON_GetObjectItem(query, "alias"); int32_t aliasBuffer = -1; if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring); diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 2c9eca11deb51acc8cc25965b83cc686ac04c6e5..8c223a15000d47becaf92fde34c92edb3f7ce707 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -86,7 +86,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; - int32_t num_fields = taos_num_fields(result); + int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); if (num_fields == 0) { return false; @@ -101,7 +101,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, // such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d) int32_t dataFields = -1; int32_t groupFields = -1; - bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; + bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; if (hasTimestamp) { dataFields = 1; if (num_fields > 2) groupFields = num_fields - 1; @@ -125,15 +125,15 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, cmd->numOfRows--; continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // for group by if (groupFields != -1) { - char target[HTTP_GC_TARGET_SIZE] = {0}; + char target[HTTP_GC_TARGET_SIZE] = {0}; int32_t len; - len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); - for (int32_t i = dataFields + 1; istate) { + switch (gzip->state) { case EHTTP_GZIP_READY: { inflateEnd(gzip->gzip); } break; - default: break; + default: + break; } if (gzip->gzip) { free(gzip->gzip); @@ -60,43 +59,43 @@ static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) { gzip->state = EHTTP_GZIP_CLOSED; } -ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { - ehttp_gzip_t *gzip = (ehttp_gzip_t*)calloc(1, sizeof(*gzip)); +ehttp_gzip_t *ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { + ehttp_gzip_t *gzip = (ehttp_gzip_t *)calloc(1, sizeof(*gzip)); if (!gzip) return NULL; do { - gzip->conf = conf; - gzip->callbacks = callbacks; - gzip->arg = arg; + gzip->conf = conf; + gzip->callbacks = callbacks; + gzip->arg = arg; if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data; - gzip->gzip = (z_stream*)calloc(1, sizeof(*gzip->gzip)); + gzip->gzip = (z_stream *)calloc(1, sizeof(*gzip->gzip)); if (gzip->conf.get_header) { - gzip->header = (gz_header*)calloc(1, sizeof(*gzip->header)); + gzip->header = (gz_header *)calloc(1, sizeof(*gzip->header)); } - if (gzip->conf.chunk_size<=0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; - gzip->chunk = (char*)malloc(gzip->conf.chunk_size); + if (gzip->conf.chunk_size <= 0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; + gzip->chunk = (char *)malloc(gzip->conf.chunk_size); if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break; - gzip->gzip->zalloc = Z_NULL; - gzip->gzip->zfree = Z_NULL; - gzip->gzip->opaque = Z_NULL; - - // 863 windowBits can also be greater than 15 for optional gzip decoding. Add - // 864 32 to windowBits to enable zlib and gzip decoding with automatic header - // 865 detection, or add 16 to decode only the gzip format (the zlib format will - // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see - // 868 below), inflate() will not automatically decode concatenated gzip streams. - // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state - // 870 would need to be reset to continue decoding a subsequent gzip stream. - int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS + gzip->gzip->zalloc = Z_NULL; + gzip->gzip->zfree = Z_NULL; + gzip->gzip->opaque = Z_NULL; + + // 863 windowBits can also be greater than 15 for optional gzip decoding. Add + // 864 32 to windowBits to enable zlib and gzip decoding with automatic header + // 865 detection, or add 16 to decode only the gzip format (the zlib format will + // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + // 868 below), inflate() will not automatically decode concatenated gzip streams. + // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state + // 870 would need to be reset to continue decoding a subsequent gzip stream. + int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS if (ret != Z_OK) break; if (gzip->header) { ret = inflateGetHeader(gzip->gzip, gzip->header); } if (ret != Z_OK) break; - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; gzip->state = EHTTP_GZIP_READY; return gzip; } while (0); @@ -105,7 +104,7 @@ ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_ return NULL; } -ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); +ehttp_gzip_t *ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); void ehttp_gzip_destroy(ehttp_gzip_t *gzip) { ehttp_gzip_cleanup(gzip); @@ -129,16 +128,16 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { } if (ret != Z_OK && ret != Z_STREAM_END) return -1; - if (gzip->gzip->avail_out>0) { - if (ret!=Z_STREAM_END) continue; + if (gzip->gzip->avail_out > 0) { + if (ret != Z_STREAM_END) continue; } - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; } return 0; @@ -147,21 +146,20 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { if (gzip->state != EHTTP_GZIP_READY) return -1; - gzip->gzip->next_in = NULL; - gzip->gzip->avail_in = 0; + gzip->gzip->next_in = NULL; + gzip->gzip->avail_in = 0; int32_t ret; ret = inflate(gzip->gzip, Z_FINISH); if (ret != Z_STREAM_END) return -1; - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = NULL; - gzip->gzip->avail_out = 0; + gzip->gzip->next_out = NULL; + gzip->gzip->avail_out = 0; return 0; } - diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 7c565075147c03cc2d65055c5d4d4773424ab927..ad79e24061d251351bc73790ac341c9dc31b29b6 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -48,7 +48,7 @@ bool httpProcessData(HttpContext* pContext) { /* * httpCloseContextByApp has been called when parsing the error */ - //httpCloseContextByApp(pContext); + // httpCloseContextByApp(pContext); } else { httpProcessRequest(pContext); } diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 1aa6cfac4bf9414e6aa8004ca1bb120b2d40feba..b120496898c12e4f189337387a5f9fad1fadccbe 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -44,20 +44,21 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t int32_t writeLen = 0; do { - if (pContext->fd > 2){ + if (pContext->fd > 2) { len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL); - } - else { + } else { return sz; } if (len < 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), + countWait); if (++countWait > HTTP_WRITE_RETRY_TIMES) break; taosMsleep(HTTP_WRITE_WAIT_TIME_MS); continue; } else if (len == 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno)); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, + strerror(errno)); break; } else { countWait = 0; @@ -80,7 +81,7 @@ int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) return writeSz; } -int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32_t sz) { +int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz) { int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz, @@ -92,8 +93,8 @@ int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32 int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { int32_t remain = 0; - char sLen[24]; - uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); + char sLen[24]; + int32_t srcLen = (int32_t)(buf->lst - buf->buf); if (buf->pContext->fd <= 0) { httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); @@ -113,21 +114,21 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int32_t len = sprintf(sLen, "%" PRIx64 "\r\n", srcLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, - srcLen, buf->buf); + int32_t len = sprintf(sLen, "%x\r\n", srcLen); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, + buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int32_t)srcLen); + remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen); } } else { - char compressBuf[JSON_BUFFER_SIZE] = {0}; + char compressBuf[JSON_BUFFER_SIZE] = {0}; int32_t compressBufLen = JSON_BUFFER_SIZE; int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast); if (ret == 0) { if (compressBufLen > 0) { int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", - buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext, + buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); } else { @@ -136,7 +137,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { remain = 0; // there is no data to dump. } } else { - httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s", + httpError("context:%p, fd:%d, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf); remain = 0; } @@ -154,8 +155,8 @@ void httpWriteJsonBufHead(JsonBuf* buf) { buf->pContext->fd = -1; } - char msg[1024] = {0}; - int32_t len = -1; + char msg[1024] = {0}; + int32_t len = -1; if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], @@ -256,16 +257,16 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) { } void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[35] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[35] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } time_t tt = t / precision; ptm = localtime(&tt); - int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); + int32_t length = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); if (us) { length += snprintf(ts + length, 8, ".%06" PRId64, t % precision); } else { @@ -276,9 +277,9 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { } void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[40] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[40] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index b84483453736aac7693c9ec671ec5a27c4178072..4ce54a8ee630579499fe498e5d8c155bb5916eea 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -130,7 +130,7 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { static void httpClearString(HttpString *str) { if (str->str) { str->str[0] = '\0'; - str->pos = 0; + str->pos = 0; } } @@ -153,7 +153,7 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target for (int32_t i = 0; i < HTTP_MAX_URL; i++) { char *pSeek = strchr(pStart, '/'); if (pSeek == NULL) { - (void)httpAppendString(pParser->path + i, pStart, strlen(pStart)); + (void)httpAppendString(pParser->path + i, pStart, (int32_t)strlen(pStart)); break; } else { (void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); @@ -229,7 +229,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const return 0; } - else if (strncasecmp(key, "Connection: ", 12) == 0) { + else if (strncasecmp(key, "Connection", 10) == 0) { if (strncasecmp(val, "Keep-Alive", 10) == 0) { parser->keepAlive = HTTP_KEEPALIVE_ENABLE; } else { @@ -237,7 +237,6 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); } - #if 0 else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { @@ -246,7 +245,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } - #endif +#endif else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { if (strstr(val, "gzip")) { @@ -272,20 +271,17 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } else if (0 == strcasecmp(key, "Authorization")) { - char * t = NULL; - char * s = NULL; + char t[6] = {0}; + char s[129] = {0}; int32_t bytes = 0; - int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); - if (n == 2 && t && s && bytes == strlen(val)) { + int32_t n = sscanf(val, "%5s %128s%n", t, s, &bytes); + if (n == 2 && t[0] && s[0] && bytes == strlen(val)) { if (strcmp(t, "Basic") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_BASIC_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH); return -1; @@ -293,13 +289,10 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const return 0; } else if (strcmp(t, "Taosd") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_TAOSD_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH); return -1; @@ -309,16 +302,12 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE); - free(t); - free(s); return -1; } } else { parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT); - free(t); - free(s); return -1; } } @@ -349,7 +338,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { newSize = MIN(newSize, HTTP_BUFFER_SIZE); buf->str = realloc(buf->str, newSize); buf->size = newSize; - + if (buf->str == NULL) { httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, buf->size); httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); @@ -410,9 +399,7 @@ static int32_t httpPopStack(HttpParser *parser) { return 0; } -static void httpClearStack(HttpStack *stack) { - stack->pos = 0; -} +static void httpClearStack(HttpStack *stack) { stack->pos = 0; } static int32_t httpCleanupStack(HttpStack *stack) { free(stack->stacks); @@ -451,7 +438,7 @@ void httpInitParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpClearStack(&parser->stacks); httpClearString(&parser->str); httpClearString(&parser->body); @@ -497,7 +484,7 @@ void httpDestroyParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpCleanupStack(&parser->stacks); httpCleanupString(&parser->str); httpCleanupString(&parser->body); @@ -513,42 +500,53 @@ void httpDestroyParser(HttpParser *parser) { free(parser); } -#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) +#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) char *httpDecodeUrl(const char *enc) { - int32_t ok = 1; + int32_t ok = 1; HttpString str = {0}; while (*enc) { char *p = strchr(enc, '%'); if (!p) break; int32_t hex, cnt; - int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt); - if (n!=1 && cnt !=2) { ok = 0; break; } - if (httpAppendString(&str, enc, p-enc)) { ok = 0; break; } + int32_t n = sscanf(p + 1, "%2x%n", &hex, &cnt); + if (n != 1 && cnt != 2) { + ok = 0; + break; + } + if (httpAppendString(&str, enc, (int32_t)(p - enc))) { + ok = 0; + break; + } char c = (char)hex; - if (httpAppendString(&str, &c, 1)) { ok = 0; break; } - enc = p+3; + if (httpAppendString(&str, &c, 1)) { + ok = 0; + break; + } + enc = p + 3; } char *dec = NULL; if (ok && *enc) { - if (httpAppendString(&str, enc, strlen(enc))) { ok = 0; } + if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { + ok = 0; + } } if (ok) { dec = str.str; str.str = NULL; } - httpCleanupString(&str); + //httpCleanupString(&str); return dec; } static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) { - HttpParser *parser = (HttpParser*)arg; + HttpParser *parser = (HttpParser *)arg; httpOnBody(parser, buf, len); } static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') { if (httpAppendString(&parser->str, &c, 1)) { @@ -570,7 +568,7 @@ static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (parser->str.pos == 1) { if (c == 'T' && parser->str.str[0] == 'H') { @@ -608,7 +606,7 @@ static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STAT static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -637,7 +635,7 @@ static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (!isspace(c) && c != '\r' && c != '\n') { if (httpAppendString(&parser->str, &c, 1)) { @@ -648,7 +646,7 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c } break; } - parser->target = strdup(parser->str.str); + parser->target = httpDecodeUrl(parser->str.str); if (!parser->target) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -664,10 +662,10 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { const char *prefix = "HTTP/1."; - int32_t len = strlen(prefix); + int32_t len = (int32_t)strlen(prefix); if (parser->str.pos < len) { if (prefix[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -717,6 +715,10 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, if (parser->method) { ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version); + if (parser->target) { + free(parser->target); + parser->target = NULL; + } } httpClearString(&parser->str); @@ -727,7 +729,7 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == ' ') { httpPopStack(parser); @@ -742,7 +744,7 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -767,7 +769,7 @@ static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE stat static int32_t httpParserOnReasonPhrase(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { parser->reasonPhrase = strdup(parser->str.str); @@ -808,10 +810,10 @@ static int32_t httpParserPostProcess(HttpParser *parser) { static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { - const char *s = "\r\n"; - int32_t len = strlen(s); + const char *s = "\r\n"; + int32_t len = (int32_t)strlen(s); if (s[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; @@ -838,7 +840,7 @@ static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, con static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { httpPopStack(parser); @@ -876,7 +878,7 @@ static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -888,7 +890,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state break; } if (c == ':') { - parser->key = strdup(parser->str.str); + parser->key = strdup(parser->str.str); if (!parser->key) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -908,7 +910,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.pos > 0)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -935,10 +937,10 @@ static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; - int32_t bytes; - int32_t len; - int32_t n; + int32_t ok = 0; + int32_t bytes; + int32_t len; + int32_t n; do { if (isxdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -985,7 +987,7 @@ static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); @@ -1019,7 +1021,7 @@ static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { ok = -1; httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -1029,8 +1031,8 @@ static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, cons } static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; + HttpContext * pContext = parser->pContext; + int32_t ok = 0; HTTP_PARSER_STATE state = httpTopStack(parser); do { if (state == HTTP_PARSER_BEGIN) { @@ -1119,9 +1121,9 @@ static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len) { HttpContext *pContext = parser->pContext; - const char *p = buf; - int32_t ret = 0; - int32_t i = 0; + const char * p = buf; + int32_t ret = 0; + int32_t i = 0; while (i < len) { int32_t again = 0; diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 1c039abb4d4cd546f6673648217a2410f57ae8e7..7f7ce404600b96b0efe8385ea0aed7be412a6ba0 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -38,16 +38,16 @@ typedef struct { } SHttpWorkerPool; typedef struct { - void * param; - void * result; - int32_t code; - int32_t rows; + void * param; + void * result; + int32_t code; + int32_t rows; FHttpResultFp fp; } SHttpResult; static SHttpWorkerPool tsHttpPool; -static taos_qset tsHttpQset; -static taos_queue tsHttpQueue; +static taos_qset tsHttpQset; +static taos_queue tsHttpQueue; void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) { if (tsHttpQueue != NULL) { @@ -59,7 +59,9 @@ void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int3 pMsg->fp = fp; taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg); } else { - (*fp)(param, result, code, rows); + taos_stop_query(result); + taos_free_result(result); + //(*fp)(param, result, code, rows); } } @@ -105,7 +107,7 @@ static bool httpAllocateResultQueue() { httpDebug("http result worker:%d is launched, total:%d", pWorker->workerId, tsHttpPool.num); } - httpInfo("http result queue is opened"); + httpInfo("http result queue is opened"); return true; } @@ -134,14 +136,14 @@ void httpCleanupResultQueue() { for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsHttpQset); } } for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 755dad2d850c24192f5eee35c49492378ba8fb73..063f2bb04e4760d683434d9e6a600791c2a57888 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -50,9 +50,16 @@ static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char * char head[512] = {0}; char body[512] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[pContext->parser->httpVersion], - httpCode, httpCodeStr, httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[httpVersion], httpCode, + httpCodeStr, httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); @@ -67,7 +74,7 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { httpCode = 404; else if (errNo == TSDB_CODE_HTTP_UNSUPPORT_URL) httpCode = 404; - else if (errNo == TSDB_CODE_HTTP_INVLALID_URL) + else if (errNo == TSDB_CODE_HTTP_INVALID_URL) httpCode = 404; else if (errNo == TSDB_CODE_HTTP_NO_ENOUGH_MEMORY) httpCode = 507; @@ -136,7 +143,7 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { else httpCode = 400; - if (pContext->parser->httpCode != 0) { + if (pContext->parser && pContext->parser->httpCode != 0) { httpCode = pContext->parser->httpCode; } @@ -153,8 +160,9 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { if (temp[i] == '\"') { temp[i] = '\''; } else if (temp[i] == '\n') { - temp[i] = ' '; - } else {} + temp[i] = ' '; + } else { + } } httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_SQL & 0XFFFF, temp); @@ -164,9 +172,16 @@ void httpSendSuccResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->parser->httpVersion], - httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[httpVersion], + httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); @@ -177,9 +192,16 @@ void httpSendOptionResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->parser->httpVersion], - httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[httpVersion], + httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); diff --git a/src/plugins/http/src/httpRestHandle.c b/src/plugins/http/src/httpRestHandle.c index 0a28c431ef313d36bd54b686e583e0dfb2313c0f..8728b9e37a1561dd545c6878756dcfbc909a27fd 100644 --- a/src/plugins/http/src/httpRestHandle.c +++ b/src/plugins/http/src/httpRestHandle.c @@ -72,11 +72,11 @@ bool restGetUserFromUrl(HttpContext* pContext) { bool restGetPassFromUrl(HttpContext* pContext) { HttpParser* pParser = pContext->parser; - if (pParser->path[REST_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) { + if (pParser->path[REST_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, TSDB_PASSWORD_LEN); + tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, HTTP_PASSWORD_LEN); return true; } @@ -95,7 +95,6 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { return false; } - /* * for async test * @@ -142,6 +141,6 @@ bool restProcessRequest(struct HttpContext* pContext) { } else { } - httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVLALID_URL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_URL); return false; } diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index a5b156bffc23ea3cdb042f456a327895fc0bdaa6..61a5a361c4865d6b3dbac73773cc0b8cfc562e56 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -35,7 +35,7 @@ void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t // data row array end httpJsonToken(jsonBuf, JsonArrEnd); - cmd->numOfRows = affect_rows; + cmd->numOfRows = 1; } void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) { @@ -75,6 +75,44 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) // head array end httpJsonToken(jsonBuf, JsonArrEnd); + // column_meta begin + httpJsonItemToken(jsonBuf); + httpJsonPairHead(jsonBuf, REST_JSON_HEAD_INFO, REST_JSON_HEAD_INFO_LEN); + // column_meta array begin + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + if (num_fields == 0) { + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + httpJsonItemToken(jsonBuf); + httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, TSDB_DATA_TYPE_INT); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, 4); + + httpJsonToken(jsonBuf, JsonArrEnd); + } else { + for (int32_t i = 0; i < num_fields; ++i) { + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + httpJsonItemToken(jsonBuf); + httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name)); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, fields[i].type); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, fields[i].bytes); + + httpJsonToken(jsonBuf, JsonArrEnd); + } + } + + // column_meta array end + httpJsonToken(jsonBuf, JsonArrEnd); + // data begin httpJsonItemToken(jsonBuf); httpJsonPairHead(jsonBuf, REST_JSON_DATA, REST_JSON_DATA_LEN); @@ -83,7 +121,8 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) httpJsonToken(jsonBuf, JsonArrStt); } -bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, int32_t timestampFormat) { +bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, + int32_t timestampFormat) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; @@ -95,7 +134,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, if (row == NULL) { continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // data row array begin httpJsonItemToken(jsonBuf); @@ -131,15 +170,17 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, (char*)row[i], length[i]); + httpJsonStringForTransMean(jsonBuf, (char *)row[i], length[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) { - httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } else if (timestampFormat == REST_TIMESTAMP_FMT_TIMESTAMP) { httpJsonInt64(jsonBuf, *((int64_t *)row[i])); } else { - httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } break; default: @@ -148,8 +189,8 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - cmd->numOfRows ++; + httpJsonToken(jsonBuf, JsonArrEnd); + cmd->numOfRows++; if (pContext->fd <= 0) { httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user); @@ -168,15 +209,15 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); } bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); } bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); } void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4896d50c6cabf6d924161c97f7fe020fe81e2ebf..a5f40fdc4cf89f6322d310ca2093336862a9045d 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -25,34 +25,70 @@ #include "httpResp.h" #include "httpUtil.h" -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif - static bool httpReadData(HttpContext *pContext); -static void httpStopThread(HttpThread* pThread) { +#ifdef __APPLE__ +static int sv_dummy = 0; +#endif + +static void httpStopThread(HttpThread *pThread) { pThread->stop = true; // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed - struct epoll_event event = { .events = EPOLLIN }; - eventfd_t fd = eventfd(1, 0); +#ifdef __APPLE__ + int sv[2]; + sv[0] = sv[1] = -1; + int r = socketpair(PF_LOCAL, SOCK_STREAM, 0, sv); + do { + if (r) break; + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &sv_dummy; + pThread->stop = true; + r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev); + if (r) break; + if (1 != send(sv[1], "1", 1, 0)) { + r = -1; + break; + } + } while (0); + if (r) { + pthread_cancel(pThread->thread); + } +#else + struct epoll_event event = {.events = EPOLLIN}; + eventfd_t fd = eventfd(1, 0); if (fd == -1) { - httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pThread->stop = true; pthread_cancel(pThread->thread); } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } +#endif // __APPLE__ pthread_join(pThread->thread, NULL); + +#ifdef __APPLE__ + if (sv[0] != -1) { + close(sv[0]); + sv[0] = -1; + } + if (sv[1] != -1) { + close(sv[1]); + sv[1] = -1; + } +#else // __APPLE__ if (fd != -1) { - close(fd); + taosCloseSocket(fd); } +#endif // __APPLE__ - close(pThread->pollFd); + EpollClose(pThread->pollFd); pthread_mutex_destroy(&(pThread->threadMutex)); } @@ -60,9 +96,12 @@ void httpCleanUpConnect() { HttpServer *pServer = &tsHttpServer; if (pServer->pThreads == NULL) return; - pthread_join(pServer->thread, NULL); + if (taosCheckPthreadValid(pServer->thread)) { + pthread_join(pServer->thread, NULL); + } + for (int32_t i = 0; i < pServer->numOfThreads; ++i) { - HttpThread* pThread = pServer->pThreads + i; + HttpThread *pThread = pServer->pThreads + i; if (pThread != NULL) { httpStopThread(pThread); } @@ -72,15 +111,12 @@ void httpCleanUpConnect() { } static void httpProcessHttpData(void *param) { - HttpServer *pServer = &tsHttpServer; - HttpThread *pThread = (HttpThread *)param; + HttpServer * pServer = &tsHttpServer; + HttpThread * pThread = (HttpThread *)param; HttpContext *pContext; int32_t fdNum; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); while (1) { struct epoll_event events[HTTP_MAX_EVENTS]; @@ -93,11 +129,20 @@ static void httpProcessHttpData(void *param) { if (fdNum <= 0) continue; for (int32_t i = 0; i < fdNum; ++i) { +#ifdef __APPLE__ + if (events[i].data.ptr == &sv_dummy) { + // no need to drain the recv buffer of sv[0] + // since there's only one time to send at most 1 byte to sv[0] + // btw, pThread->stop shall be already set, thus never reached here + httpDebug("if you see this line, there's internal logic error"); + continue; + } +#endif // __APPLE__ pContext = httpGetContext(events[i].data.ptr); if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); - //epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); - //taosClose(events[i].data.fd); + // epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + // taosClose(events[i].data.fd); continue; } @@ -154,7 +199,7 @@ static void httpProcessHttpData(void *param) { } static void *httpAcceptHttpConnection(void *arg) { - int32_t connFd = -1; + SOCKET connFd = -1; struct sockaddr_in clientAddr; int32_t threadId = 0; HttpServer * pServer = &tsHttpServer; @@ -162,10 +207,7 @@ static void *httpAcceptHttpConnection(void *arg) { HttpContext * pContext = NULL; int32_t totalFds = 0; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort); @@ -180,7 +222,12 @@ static void *httpAcceptHttpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(clientAddr); - connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + connFd = accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + if (pServer->stop) { + httpDebug("http server:%s socket stop, exiting...", pServer->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { httpDebug("http server:%s socket was shutdown, exiting...", pServer->label); @@ -202,7 +249,7 @@ static void *httpAcceptHttpConnection(void *arg) { taosCloseSocket(connFd); continue; } -#endif +#endif taosKeepTcpAlive(connFd); taosSetNonblocking(connFd, 1); @@ -212,22 +259,22 @@ static void *httpAcceptHttpConnection(void *arg) { pContext = httpCreateContext(connFd); if (pContext == NULL) { - httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, taosInetNtoa(clientAddr.sin_addr), - htons(clientAddr.sin_port)); + httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, + taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosCloseSocket(connFd); continue; } pContext->pThread = pThread; sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); - + struct epoll_event event; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; event.data.ptr = pContext; if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); - taosClose(pContext->fd); + taosCloseSocket(pContext->fd); httpReleaseContext(pContext, true); continue; } @@ -242,7 +289,7 @@ static void *httpAcceptHttpConnection(void *arg) { threadId = threadId % pServer->numOfThreads; } - close(pServer->fd); + taosCloseSocket(pServer->fd); return NULL; } @@ -265,8 +312,8 @@ bool httpInitConnect() { return false; } - pThread->pollFd = epoll_create(HTTP_MAX_EVENTS); // size does not matter - if (pThread->pollFd < 0) { + pThread->pollFd = (EpollFd)epoll_create(HTTP_MAX_EVENTS); // size does not matter + if (pThread->pollFd <= 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); return false; @@ -278,7 +325,7 @@ bool httpInitConnect() { if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) { httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label, strerror(errno)); - pthread_mutex_destroy(&(pThread->threadMutex)); + pthread_mutex_destroy(&(pThread->threadMutex)); return false; } pthread_attr_destroy(&thattr); @@ -309,7 +356,7 @@ static bool httpReadData(HttpContext *pContext) { } if (pParser->parsed) { - httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); + httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); return false; } diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index a96e4433b065885a18b8c0b4f35e2a1925125fc3..2e1ee7df2fe53d835ee5f3b61bd7619a9121edce 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -35,7 +35,8 @@ void httpCreateSession(HttpContext *pContext, void *taos) { session.refCount = 1; int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); + pContext->session = + taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); // void *temp = pContext->session; // taosCacheRelease(server->sessionCache, (void **)&temp, false); @@ -56,7 +57,7 @@ static void httpFetchSessionImp(HttpContext *pContext) { HttpServer *server = &tsHttpServer; pthread_mutex_lock(&server->serverMutex); - char sessionId[HTTP_SESSION_ID_LEN]; + char sessionId[HTTP_SESSION_ID_LEN]; int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); @@ -107,7 +108,7 @@ static void httpDestroySession(void *data) { void httpCleanUpSessions() { if (tsHttpServer.sessionCache != NULL) { SCacheObj *cache = tsHttpServer.sessionCache; - httpInfo("session cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable)); + httpInfo("session cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable)); taosCacheCleanup(tsHttpServer.sessionCache); tsHttpServer.sessionCache = NULL; } diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 564f555c403b6f5b3deb5832da6505722104eff8..4e9b54b7bd2b25b2a8bbdd29e6d54ae314b7519d 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -64,7 +64,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32 } taos_free_result(result); - + if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { (encode->stopJsonFp)(pContext, singleCmd); } @@ -82,7 +82,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - HttpSqlCmds *multiCmds = pContext->multiCmds; + HttpSqlCmds * multiCmds = pContext->multiCmds; HttpEncodeMethod *encode = pContext->encodeMethod; HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; @@ -181,7 +181,7 @@ void httpProcessMultiSql(HttpContext *pContext) { char *sql = httpGetCmdsString(pContext, cmd->sql); httpTraceL("context:%p, fd:%d, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->user, multiCmds->pos, sql); - taosNotePrintHttp(sql); + nPrintHttp("%s", sql); taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext); } @@ -269,8 +269,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj)); } else { - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, - pContext->user, tstrerror(code), pObj); + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, + tstrerror(code), pObj); httpSendErrorResp(pContext, code); } taos_free_result(result); @@ -329,7 +329,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) { } httpTraceL("context:%p, fd:%d, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->user, sql); - taosNotePrintHttp(sql); + nPrintHttp("%s", sql); taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext); } @@ -381,7 +381,7 @@ void httpExecCmd(HttpContext *pContext) { void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { HttpContext *pContext = param; taos_free_result(result); - + if (pContext == NULL) return; if (code < 0) { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 3b8858b62e1297f0db5ece60c59a26e376b546f3..203db218953b6d9859a4e60b589ce01e730e2773 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -37,7 +37,6 @@ void opInitHandle(HttpServer* pServer) {} #endif HttpServer tsHttpServer; -void taosInitNote(int32_t numOfNoteLines, int32_t maxNotes, char* lable); int32_t httpInitSystem() { strcpy(tsHttpServer.label, "rest"); @@ -48,9 +47,6 @@ int32_t httpInitSystem() { pthread_mutex_init(&tsHttpServer.serverMutex, NULL); - if (tsHttpEnableRecordSql != 0) { - taosInitNote(tsNumOfLogLines / 10, 1, (char*)"http_note"); - } restInitHandle(&tsHttpServer); adminInitHandle(&tsHttpServer); gcInitHandle(&tsHttpServer); @@ -93,7 +89,17 @@ int32_t httpStartSystem() { void httpStopSystem() { tsHttpServer.status = HTTP_SERVER_CLOSING; + tsHttpServer.stop = 1; +#ifdef WINDOWS + closesocket(tsHttpServer.fd); +#elif __APPLE__ + if (tsHttpServer.fd!=-1) { + close(tsHttpServer.fd); + tsHttpServer.fd = -1; + } +#else shutdown(tsHttpServer.fd, SHUT_RD); +#endif tgCleanupHandle(); } @@ -109,10 +115,8 @@ void httpCleanUpSystem() { pthread_mutex_destroy(&tsHttpServer.serverMutex); tfree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; - + tsHttpServer.status = HTTP_SERVER_CLOSED; } -int32_t httpGetReqCount() { - return atomic_exchange_32(&tsHttpServer.requestNum, 0); -} +int32_t httpGetReqCount() { return atomic_exchange_32(&tsHttpServer.requestNum, 0); } diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index e2b57b87bb3a10ee50edbf55ae2590289f940bf6..c1d006ff5a0c2cb5037ee6833b58d2ba749196e5 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -137,8 +137,8 @@ void tgInitSchemas(int32_t size) { } void tgParseSchemaMetric(cJSON *metric) { - STgSchema schema = {0}; - bool parsedOk = true; + STgSchema schema = {0}; + bool parsedOk = true; // name cJSON *name = cJSON_GetObjectItem(metric, "name"); @@ -186,7 +186,7 @@ void tgParseSchemaMetric(cJSON *metric) { schema.tbName = calloc(tbnameLen + 1, 1); strcpy(schema.tbName, tbname->valuestring); - // fields + // fields cJSON *fields = cJSON_GetObjectItem(metric, "fields"); if (fields == NULL) { goto ParseEnd; @@ -227,14 +227,14 @@ ParseEnd: } } -int32_t tgParseSchema(const char *content, char*fileName) { +int32_t tgParseSchema(const char *content, char *fileName) { cJSON *root = cJSON_Parse(content); if (root == NULL) { httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content); return -1; } int32_t size = 0; - cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); + cJSON * metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { size = cJSON_GetArraySize(metrics); if (size <= 0) { @@ -276,8 +276,8 @@ int32_t tgReadSchema(char *fileName) { rewind(fp); char * content = (char *)calloc(contentSize + 1, 1); - int32_t result = fread(content, 1, contentSize, fp); - + int32_t result = (int32_t)fread(content, 1, contentSize, fp); + if (result != contentSize) { httpError("failed to read telegraf schema file:%s", fileName); fclose(fp); @@ -296,7 +296,7 @@ int32_t tgReadSchema(char *fileName) { } void tgInitHandle(HttpServer *pServer) { - char fileName[TSDB_FILENAME_LEN*2] = {0}; + char fileName[TSDB_FILENAME_LEN * 2] = {0}; sprintf(fileName, "%s/taos.telegraf.cfg", configDir); if (tgReadSchema(fileName) <= 0) { tgFreeSchemas(); @@ -308,9 +308,7 @@ void tgInitHandle(HttpServer *pServer) { httpAddMethod(pServer, &tgDecodeMethod); } -void tgCleanupHandle() { - tgFreeSchemas(); -} +void tgCleanupHandle() { tgFreeSchemas(); } bool tgGetUserFromUrl(HttpContext *pContext) { HttpParser *pParser = pContext->parser; @@ -324,7 +322,7 @@ bool tgGetUserFromUrl(HttpContext *pContext) { bool tgGetPassFromUrl(HttpContext *pContext) { HttpParser *pParser = pContext->parser; - if (pParser->path[TG_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) { + if (pParser->path[TG_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) { return false; } @@ -357,7 +355,7 @@ char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) { bool schemaMatched = true; for (int32_t f = 0; f < schema->fieldNum; ++f) { char *fieldName = schema->fields[f]; - bool fieldMatched = false; + bool fieldMatched = false; for (int32_t i = 0; i < fieldsSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); @@ -469,9 +467,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(tag->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE); @@ -540,9 +538,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { return false; } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(field->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE); @@ -578,8 +576,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { table_cmd->cmdType = HTTP_CMD_TYPE_INSERT; // order by tag name - cJSON *orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; - int32_t orderTagsLen = 0; + cJSON * orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; + int32_t orderTagsLen = 0; for (int32_t i = 0; i < tagsSize; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); orderedTags[orderTagsLen++] = tag; @@ -603,7 +601,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { if (tsTelegrafUseFieldNum == 0) { table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s", stname); } else { - table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); + table_cmd->stable = stable_cmd->stable = + httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); } table_cmd->stable = stable_cmd->stable = httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable)); @@ -627,9 +626,11 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // table name if (tsTelegrafUseFieldNum == 0) { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); } else { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); } for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 39168ee96dc0ef83dc329614f63543d2c6c7fdfa..399a33954d5670a7e928f856a898dcde1e4ac4eb 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -29,7 +29,7 @@ bool httpCheckUsedbSql(char *sql) { return false; } -void httpTimeToString(time_t t, char *buf, int32_t buflen) { +void httpTimeToString(int32_t t, char *buf, int32_t buflen) { memset(buf, 0, (size_t)buflen); char ts[32] = {0}; @@ -37,7 +37,7 @@ void httpTimeToString(time_t t, char *buf, int32_t buflen) { time_t tt = t / 1000; ptm = localtime(&tt); strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03ld", ts, t % 1000); + sprintf(buf, "%s.%03d", ts, t % 1000); } int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { @@ -160,8 +160,7 @@ bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferS free(multiCmds->cmds); multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd)); if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, - pContext->user, cmdSize); + httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); return false; } multiCmds->maxSize = (int16_t)cmdSize; @@ -240,6 +239,10 @@ JsonBuf *httpMallocJsonBuf(HttpContext *pContext) { pContext->jsonBuf = (JsonBuf *)malloc(sizeof(JsonBuf)); } + if (!pContext->jsonBuf->pContext) { + pContext->jsonBuf->pContext = pContext; + } + return pContext->jsonBuf; } @@ -350,74 +353,78 @@ char *httpGetCmdsString(HttpContext *pContext, int32_t pos) { } int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) { - int32_t err = 0; + int32_t err = 0; z_stream gzipStream = {0}; static char dummyHead[2] = { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, + 0x8 + 0x7 * 0x10, + (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, }; - gzipStream.zalloc = (alloc_func) 0; - gzipStream.zfree = (free_func) 0; - gzipStream.opaque = (voidpf) 0; - gzipStream.next_in = (Bytef *) srcData; + gzipStream.zalloc = (alloc_func)0; + gzipStream.zfree = (free_func)0; + gzipStream.opaque = (voidpf)0; + gzipStream.next_in = (Bytef *)srcData; gzipStream.avail_in = 0; - gzipStream.next_out = (Bytef *) destData; + gzipStream.next_out = (Bytef *)destData; if (inflateInit2(&gzipStream, 47) != Z_OK) { return -1; } while (gzipStream.total_out < *nDestData && gzipStream.total_in < nSrcData) { - gzipStream.avail_in = gzipStream.avail_out = nSrcData; //1 + gzipStream.avail_in = gzipStream.avail_out = nSrcData; // 1 if ((err = inflate(&gzipStream, Z_NO_FLUSH)) == Z_STREAM_END) { break; } if (err != Z_OK) { if (err == Z_DATA_ERROR) { - gzipStream.next_in = (Bytef *) dummyHead; + gzipStream.next_in = (Bytef *)dummyHead; gzipStream.avail_in = sizeof(dummyHead); if ((err = inflate(&gzipStream, Z_NO_FLUSH)) != Z_OK) { return -2; } - } else return -3; + } else { + return -3; + } } } if (inflateEnd(&gzipStream) != Z_OK) { return -4; } - *nDestData = gzipStream.total_out; + *nDestData = (int32_t)gzipStream.total_out; return 0; } int32_t httpGzipCompressInit(HttpContext *pContext) { - pContext->gzipStream.zalloc = (alloc_func) 0; - pContext->gzipStream.zfree = (free_func) 0; - pContext->gzipStream.opaque = (voidpf) 0; - if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + pContext->gzipStream.zalloc = (alloc_func)0; + pContext->gzipStream.zfree = (free_func)0; + pContext->gzipStream.opaque = (voidpf)0; + if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != + Z_OK) { return -1; } return 0; } -int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) { +int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, + bool isTheLast) { int32_t err = 0; - int32_t lastTotalLen = (int32_t) (pContext->gzipStream.total_out); - pContext->gzipStream.next_in = (Bytef *) srcData; - pContext->gzipStream.avail_in = (uLong) nSrcData; - pContext->gzipStream.next_out = (Bytef *) destData; - pContext->gzipStream.avail_out = (uLong) (*nDestData); + int32_t lastTotalLen = (int32_t)(pContext->gzipStream.total_out); + pContext->gzipStream.next_in = (Bytef *)srcData; + pContext->gzipStream.avail_in = (uLong)nSrcData; + pContext->gzipStream.next_out = (Bytef *)destData; + pContext->gzipStream.avail_out = (uLong)(*nDestData); while (pContext->gzipStream.avail_in != 0) { if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) { return -1; } - int32_t cacheLen = pContext->gzipStream.total_out - lastTotalLen; + int32_t cacheLen = (int32_t)(pContext->gzipStream.total_out - lastTotalLen); if (cacheLen >= *nDestData) { return -2; } @@ -442,12 +449,12 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, } } - *nDestData = (int32_t) (pContext->gzipStream.total_out) - lastTotalLen; + *nDestData = (int32_t)(pContext->gzipStream.total_out) - lastTotalLen; return 0; } -bool httpUrlMatch(HttpContext* pContext, int32_t pos, char* cmp) { - HttpParser* pParser = pContext->parser; +bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp) { + HttpParser *pParser = pContext->parser; if (pos < 0 || pos >= HTTP_MAX_URL) { return false; diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index edea2187ea23caa5d809346a43d14f6ffaf8a309..28c62a099c0f2bea8b33a57c577bc89c7fb15aaa 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -6,12 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF (TD_LINUX) - ADD_LIBRARY(monitor ${SRC}) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) - ELSE () - TARGET_LINK_LIBRARIES(monitor taos) - ENDIF () -ENDIF () +ADD_LIBRARY(monitor ${SRC}) + +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(monitor taos_static) +ELSE () + TARGET_LINK_LIBRARIES(monitor taos) +ENDIF () diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 9443b1ce12deba75a070f25ae4f67f7887870f7d..ac80ad62509a755b5b86b65593a90a2730120df8 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -20,7 +20,6 @@ #include "tlog.h" #include "ttimer.h" #include "tutil.h" -#include "tsystem.h" #include "tscUtil.h" #include "tsclient.h" #include "dnode.h" @@ -79,8 +78,8 @@ int32_t monInitSystem() { strcpy(tsMonitor.ep, tsLocalEp); } - int len = strlen(tsMonitor.ep); - for (int i = 0; i < len; ++i) { + int32_t len = (int32_t)strlen(tsMonitor.ep); + for (int32_t i = 0; i < len; ++i) { if (tsMonitor.ep[i] == ':' || tsMonitor.ep[i] == '-' || tsMonitor.ep[i] == '.') { tsMonitor.ep[i] = '_'; } @@ -104,7 +103,9 @@ int32_t monInitSystem() { } int32_t monStartSystem() { - taos_init(); + if (taos_init()) { + return -1; + } tsMonitor.start = 1; monExecuteSQLFp = monExecuteSQL; monInfo("monitor module start"); @@ -148,7 +149,7 @@ static void *monThreadFunc(void *param) { } if (tsMonitor.state == MON_STATE_NOT_INIT) { - int code = 0; + int32_t code = 0; for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) { monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); @@ -246,7 +247,10 @@ void monStopSystem() { void monCleanupSystem() { tsMonitor.quiting = 1; monStopSystem(); - pthread_join(tsMonitor.thread, NULL); + if (taosCheckPthreadValid(tsMonitor.thread)) { + pthread_join(tsMonitor.thread, NULL); + } + if (tsMonitor.conn != NULL) { taos_close(tsMonitor.conn); tsMonitor.conn = NULL; @@ -330,7 +334,7 @@ static void monSaveSystemInfo() { pos += monBuildReqSql(sql + pos); void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - int code = taos_errno(res); + int32_t code = taos_errno(res); taos_free_result(res); if (code != 0) { diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt index cf5729d6087065bd38a5bed140ec01b1b531f454..50b0bbe8af4faeab41a7b041d6aa51747f0aab3e 100644 --- a/src/plugins/mqtt/CMakeLists.txt +++ b/src/plugins/mqtt/CMakeLists.txt @@ -18,3 +18,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(mqtt taos) ENDIF () ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(mqtt ${SRC}) + TARGET_LINK_LIBRARIES(mqtt cJson mqttc) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(mqtt taos_static) + ELSE () + TARGET_LINK_LIBRARIES(mqtt taos) + ENDIF () +ENDIF () + diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index c1024f008095be76dba0d8dd59447a489c015772..f23ac7dd86932ba42dde7c2891865f7dff546a00 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -14,3 +14,8 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(query m rt) ADD_SUBDIRECTORY(tests) ENDIF () + +IF (TD_DARWIN) + TARGET_LINK_LIBRARIES(query m) + ADD_SUBDIRECTORY(tests) +ENDIF () diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/qAggMain.h similarity index 76% rename from src/query/inc/tsqlfunction.h rename to src/query/inc/qAggMain.h index 5a923db52c6b9a7897de2b2ac72e59bf570a3a56..7122f63593c64a6bffd6f3d53cd8ef21eca800ff 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/qAggMain.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSQLFUNCTION_H -#define TDENGINE_TSQLFUNCTION_H +#ifndef TDENGINE_QAGGMAIN_H +#define TDENGINE_QAGGMAIN_H #ifdef __cplusplus extern "C" { @@ -59,30 +59,32 @@ extern "C" { #define TSDB_FUNC_FIRST_DST 25 #define TSDB_FUNC_LAST_DST 26 -#define TSDB_FUNC_INTERP 27 - -#define TSDB_FUNC_RATE 28 -#define TSDB_FUNC_IRATE 29 -#define TSDB_FUNC_SUM_RATE 30 -#define TSDB_FUNC_SUM_IRATE 31 -#define TSDB_FUNC_AVG_RATE 32 -#define TSDB_FUNC_AVG_IRATE 33 - -#define TSDB_FUNC_TID_TAG 34 -#define TSDB_FUNC_HISTOGRAM 35 -#define TSDB_FUNC_HLL 36 -#define TSDB_FUNC_MODE 37 -#define TSDB_FUNC_SAMPLE 38 -#define TSDB_FUNC_CEIL 39 -#define TSDB_FUNC_FLOOR 40 -#define TSDB_FUNC_ROUND 41 -#define TSDB_FUNC_MAVG 42 -#define TSDB_FUNC_CSUM 43 +#define TSDB_FUNC_STDDEV_DST 27 +#define TSDB_FUNC_INTERP 28 + +#define TSDB_FUNC_RATE 29 +#define TSDB_FUNC_IRATE 30 +#define TSDB_FUNC_SUM_RATE 31 +#define TSDB_FUNC_SUM_IRATE 32 +#define TSDB_FUNC_AVG_RATE 33 +#define TSDB_FUNC_AVG_IRATE 34 + +#define TSDB_FUNC_TID_TAG 35 +#define TSDB_FUNC_HISTOGRAM 36 +#define TSDB_FUNC_HLL 37 +#define TSDB_FUNC_MODE 38 +#define TSDB_FUNC_SAMPLE 39 +#define TSDB_FUNC_CEIL 40 +#define TSDB_FUNC_FLOOR 41 +#define TSDB_FUNC_ROUND 42 +#define TSDB_FUNC_MAVG 43 +#define TSDB_FUNC_CSUM 44 + #define TSDB_FUNCSTATE_SO 0x1u // single output #define TSDB_FUNCSTATE_MO 0x2u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM #define TSDB_FUNCSTATE_STREAM 0x4u // function avail for stream -#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for metric +#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for super table #define TSDB_FUNCSTATE_OF 0x10u // outer forward #define TSDB_FUNCSTATE_NEED_TS 0x20u // timestamp is required during query processing #define TSDB_FUNCSTATE_SELECTIVITY 0x40u // selectivity functions, can exists along with tag columns @@ -90,19 +92,12 @@ extern "C" { #define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF #define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF - #define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 #define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) -#define QUERY_COND_REL_PREFIX_IN "IN|" -#define QUERY_COND_REL_PREFIX_LIKE "LIKE|" - -#define QUERY_COND_REL_PREFIX_IN_LEN 3 -#define QUERY_COND_REL_PREFIX_LIKE_LEN 5 - #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 @@ -112,11 +107,10 @@ extern "C" { #define TOP_BOTTOM_QUERY_LIMIT 100 enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - FIRST_STAGE_MERGE = 0x10u, - SECONDARY_STAGE_MERGE = 0x20u, + MASTER_SCAN = 0x0u, + REVERSE_SCAN = 0x1u, + REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan + MERGE_STAGE = 0x20u, }; #define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) @@ -154,7 +148,7 @@ typedef struct SResultRowCellInfo { typedef struct SPoint1 { int64_t key; - double val; + union{double val; char* ptr;}; } SPoint1; #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) @@ -172,8 +166,8 @@ typedef struct SExtTagsInfo { // sql function runtime context typedef struct SQLFunctionCtx { - int32_t startOffset; int32_t size; // number of rows + void * pInput; // input data buffer uint32_t order; // asc|desc int16_t inputType; int16_t inputBytes; @@ -182,17 +176,16 @@ typedef struct SQLFunctionCtx { int16_t outputBytes; // size of results, determined by function and input column data type int32_t interBufBytes; // internal buffer size bool hasNull; // null value exist in current block - bool requireNull; // require null in some function + bool requireNull; // require null in some function bool stableQuery; - int16_t functionId; // function id - void * aInputElemBuf; - char * aOutputBuf; // final result output buffer, point to sdata->data - uint8_t currentStage; // record current running step, default: 0 - int64_t nStartQueryTimestamp; // timestamp range of current query when function is executed on a specific data block + int16_t functionId; // function id + char * pOutput; // final result output buffer, point to sdata->data + uint8_t currentStage; // record current running step, default: 0 + int64_t startTs; // timestamp range of current query when function is executed on a specific data block int32_t numOfParams; - tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */ - int64_t * ptsList; // corresponding timestamp array list - void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param + int64_t *ptsList; // corresponding timestamp array list + void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; tVariant tag; @@ -203,33 +196,26 @@ typedef struct SQLFunctionCtx { SPoint1 end; } SQLFunctionCtx; -typedef struct SQLAggFuncElem { - char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; - - uint8_t nAggIdx; // index of function in aAggs +typedef struct SAggFunctionInfo { + char name[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; + uint8_t index; // index of function in aAggs int8_t stableFuncId; // transfer function for super table query - uint16_t nStatus; + uint16_t status; bool (*init)(SQLFunctionCtx *pCtx); // setup the execute environment void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function - void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version + void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version, todo merge with blockwise function - // some sql function require scan data twice or more, e.g.,stddev + // some sql function require scan data twice or more, e.g.,stddev, percentile void (*xNextStep)(SQLFunctionCtx *pCtx); - /* - * finalizer must be called after all xFunction has been executed to - * generated final result. Otherwise, the value in aOutputBuf is a intern result. - */ + // finalizer must be called after all xFunction has been executed to generated final result. void (*xFinalize)(SQLFunctionCtx *pCtx); - - void (*distMergeFunc)(SQLFunctionCtx *pCtx); - - void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx); + void (*mergeFunc)(SQLFunctionCtx *pCtx); int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); -} SQLAggFuncElem; +} SAggFunctionInfo; #define GET_RES_INFO(ctx) ((ctx)->resultInfo) @@ -241,7 +227,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI #define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) #define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) -/* determine the real data need to calculated the result */ +// determine the real data need to calculated the result enum { BLK_DATA_NO_NEEDED = 0x0, BLK_DATA_STATIS_NEEDED = 0x1, @@ -250,15 +236,14 @@ enum { }; typedef struct STwaInfo { - TSKEY lastKey; int8_t hasResult; // flag to denote has value double dOutput; - double lastValue; + SPoint1 p; STimeWindow win; } STwaInfo; /* global sql function array */ -extern struct SQLAggFuncElem aAggs[]; +extern struct SAggFunctionInfo aAggs[]; extern int32_t functionCompatList[]; // compatible check array list @@ -287,4 +272,4 @@ static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t b } #endif -#endif // TDENGINE_TSQLFUNCTION_H +#endif // TDENGINE_QAGGMAIN_H diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b73f7ce3f5cad809c12fcf17e5c57a79811605fd..ec1261da0a45ad24985ebf51b9b16c2acfad7709 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -18,6 +18,7 @@ #include "os.h" #include "hash.h" +#include "qAggMain.h" #include "qFill.h" #include "qResultbuf.h" #include "qSqlparser.h" @@ -27,18 +28,40 @@ #include "tarray.h" #include "tlockfree.h" #include "tsdb.h" -#include "tsqlfunction.h" struct SColumnFilterElem; -typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); +typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -typedef struct SGroupResInfo { - int32_t groupId; - int32_t numOfDataPages; - int32_t pageId; - int32_t rowId; -} SGroupResInfo; +#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) +#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) + +#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) +#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) + +#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) + +enum { + // when query starts to execute, this status will set + QUERY_NOT_COMPLETED = 0x1u, + + /* result output buffer is full, current query is paused. + * this status is only exist in group-by clause and diff/add/division/multiply/ query. + */ + QUERY_RESBUF_FULL = 0x2u, + + /* query is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed + */ + QUERY_COMPLETED = 0x4u, + + /* when the result is not completed return to client, this status will be + * usually used in case of interval query with interpolation option + */ + QUERY_OVER = 0x8u, +}; typedef struct SResultRowPool { int32_t elemSize; @@ -72,6 +95,13 @@ typedef struct SResultRow { union {STimeWindow win; char* key;}; // start key of current time window } SResultRow; +typedef struct SGroupResInfo { + int32_t totalGroup; + int32_t currentGroup; + int32_t index; + SArray* pRows; // SArray +} SGroupResInfo; + /** * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. @@ -89,7 +119,6 @@ typedef struct SResultRowInfo { int32_t size:24; // number of result set int32_t capacity; // max capacity int32_t curIndex; // current start active index - int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key } SResultRowInfo; @@ -114,7 +143,7 @@ typedef struct STableQueryInfo { STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list - SResultRowInfo windowResInfo; + SResultRowInfo resInfo; } STableQueryInfo; typedef struct SQueryCostInfo { @@ -140,6 +169,16 @@ typedef struct SQueryCostInfo { uint64_t numOfTimeWindows; } SQueryCostInfo; +typedef struct { + int64_t vgroupLimit; + int64_t ts; +} SOrderedPrjQueryInfo; + +typedef struct { + char* tags; + SArray* pResult; // SArray +} SInterResult; + typedef struct SQuery { int16_t numOfCols; int16_t numOfTags; @@ -149,9 +188,14 @@ typedef struct SQuery { int16_t precision; int16_t numOfOutput; int16_t fillType; - int16_t checkBuffer; // check if the buffer is full during scan each block + int16_t checkResultBuf; // check if the buffer is full during scan each block SLimitVal limit; - int32_t rowSize; + + int32_t srcRowSize; // todo extract struct + int32_t resultRowSize; + int32_t maxSrcColumnSize; + int32_t tagLen; // tag value length of current query + SSqlGroupbyExpr* pGroupbyExpr; SExprInfo* pExpr1; SExprInfo* pExpr2; @@ -161,12 +205,14 @@ typedef struct SQuery { SColumnInfo* tagColList; int32_t numOfFilterCols; int64_t* fillVal; - uint32_t status; // query status + uint32_t status; // query status SResultRec rec; int32_t pos; tFilePage** sdata; STableQueryInfo* current; + int32_t numOfCheckedBlocks; // number of check data blocks + SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. SSingleColumnFilterInfo* pFilterInfo; } SQuery; @@ -178,18 +224,19 @@ typedef struct SQueryRuntimeEnv { uint16_t* offset; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; - SResultRowInfo windowResInfo; - STSBuf* pTSBuf; - STSCursor cur; + SResultRowInfo resultRowInfo; + SQueryCostInfo summary; void* pQueryHandle; void* pSecQueryHandle; // another thread for bool stableQuery; // super table query or not - bool topBotQuery; // false - bool groupbyNormalCol; // denote if this is a groupby normal column query + bool topBotQuery; // TODO used bitwise flag + bool groupbyColumn; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not bool timeWindowInterpo;// if the time window start/end required interpolation bool queryWindowIdentical; // all query time windows are identical for all tables in one group + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file @@ -199,7 +246,13 @@ typedef struct SQueryRuntimeEnv { int32_t* rowCellInfoOffset;// offset value for each row result cell info char** prevRow; - char** nextRow; + + SArray* prevResult; // intermediate result, SArray + STSBuf* pTsBuf; // timestamp filter list + STSCursor cur; + + char* tagVal; // tag value of current data block + SArithmeticSupport *sasArray; } SQueryRuntimeEnv; enum { @@ -210,14 +263,13 @@ enum { typedef struct SQInfo { void* signature; int32_t code; // error code to returned to client - int64_t owner; // if it is in execution + int64_t owner; // if it is in execution void* tsdb; SMemRef memRef; int32_t vgId; STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; -// SArray* arrTableIdInfo; SHashObj* arrTableIdInfo; int32_t groupIndex; @@ -233,6 +285,55 @@ typedef struct SQInfo { tsem_t ready; int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context + int64_t startExecTs; // start to exec timestamp + char* sql; // query sql string } SQInfo; +typedef struct SQueryParam { + char *sql; + char *tagCond; + char *tbnameCond; + char *prevResult; + SArray *pTableIdList; + SSqlFuncMsg **pExprMsg; + SSqlFuncMsg **pSecExprMsg; + SExprInfo *pExprs; + SExprInfo *pSecExprs; + + SColIndex *pGroupColIndex; + SColumnInfo *pTagColumnInfo; + SSqlGroupbyExpr *pGroupbyExpr; +} SQueryParam; + +void freeParam(SQueryParam *param); +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); +int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg, + SColumnInfo* pTagCols); +SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); +SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql); +int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, SQueryParam* param, bool isSTable); +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); + +bool isQueryKilled(SQInfo *pQInfo); +int32_t checkForQueryBuf(size_t numOfTables); +bool doBuildResCheck(SQInfo* pQInfo); +void setQueryStatus(SQuery *pQuery, int8_t status); + +bool onlyQueryTags(SQuery* pQuery); +void buildTagQueryResult(SQInfo *pQInfo); +void stableQueryImpl(SQInfo *pQInfo); +void buildTableBlockDistResult(SQInfo *pQInfo); +void tableQueryImpl(SQInfo *pQInfo); +bool isValidQInfo(void *param); + +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data); + +size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); +void setQueryKilled(SQInfo *pQInfo); +void queryCostStatis(SQInfo *pQInfo); +void freeQInfo(SQInfo *pQInfo); + +int32_t getMaximumIdleDurationSec(); + #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 385ae8854327b036cbb6ee46de70e97ae56ce25b..aa6df9279acb9e3cecf20d71f726974ee89a4030 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -68,7 +68,7 @@ typedef struct SPoint { void * val; } SPoint; -SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, +SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol, void* handle); @@ -78,15 +78,15 @@ void* taosDestroyFillInfo(SFillInfo *pFillInfo); void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput); +void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput); void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput); -int64_t getNumOfResWithFill(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); +bool taosFillHasMoreResults(SFillInfo* pFillInfo); -int32_t taosNumOfRemainRows(SFillInfo *pFillInfo); +int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); -int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point); +int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity); diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 442e61750b06d269ef48c5e74ede7646f7ac62e1..7742d151a06456a8b3c10e8b1f0aeba6203a28b7 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto); void tHistogramPrint(SHistogramInfo* pHisto); -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val); +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val); SHeapEntry* tHeapCreate(int32_t numOfEntries); void tHeapSort(SHeapEntry* pEntry, int32_t len); diff --git a/src/query/inc/qPercentile.h b/src/query/inc/qPercentile.h index c34c24c5b2883391f94d87db4f778907cc13d62c..f5b770593c1e30df8c3459676ffa42bbfc9cf0ae 100644 --- a/src/query/inc/qPercentile.h +++ b/src/query/inc/qPercentile.h @@ -16,32 +16,36 @@ #ifndef TDENGINE_QPERCENTILE_H #define TDENGINE_QPERCENTILE_H +#ifdef __cplusplus +extern "C" { +#endif + #include "qExtbuffer.h" #include "qResultbuf.h" #include "qTsbuf.h" typedef struct MinMaxEntry { union { - double dMinVal; - int32_t iMinVal; - int64_t i64MinVal; + double dMinVal; + int64_t i64MinVal; + uint64_t u64MinVal; }; union { double dMaxVal; - int32_t iMaxVal; int64_t i64MaxVal; + int64_t u64MaxVal; }; } MinMaxEntry; typedef struct { - int32_t size; - int32_t pageId; + int32_t size; + int32_t pageId; tFilePage *data; } SSlotInfo; typedef struct tMemBucketSlot { - SSlotInfo info; - MinMaxEntry range; + SSlotInfo info; + MinMaxEntry range; } tMemBucketSlot; struct tMemBucket; @@ -52,16 +56,16 @@ typedef struct tMemBucket { int16_t type; int16_t bytes; int32_t total; - int32_t elemPerPage; // number of elements for each object - int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result - int32_t bufPageSize; // disk page size - MinMaxEntry range; // value range - int32_t times; // count that has been checked for deciding the correct data value buckets. + int32_t elemPerPage; // number of elements for each object + int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result + int32_t bufPageSize; // disk page size + MinMaxEntry range; // value range + int32_t times; // count that has been checked for deciding the correct data value buckets. __compar_fn_t comparFn; - tMemBucketSlot *pSlots; + tMemBucketSlot * pSlots; SDiskbasedResultBuf *pBuffer; - __perc_hash_func_t hashFunc; + __perc_hash_func_t hashFunc; } tMemBucket; tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval); @@ -73,3 +77,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); double getPercentile(tMemBucket *pMemBucket, double percent); #endif // TDENGINE_QPERCENTILE_H + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 513ab090f921b15ca18e384643ab79f51a119a8d..bcc876c953777f465ecb761f19256a86bc375b1d 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -73,34 +73,40 @@ typedef struct SQuerySQL { SStrToken selectToken; // sql string } SQuerySQL; +typedef struct SCreatedTableInfo { + SStrToken name; // table name token + SStrToken stableName; // super table name token , for using clause + SArray *pTagNames; // create by using super table, tag name + SArray *pTagVals; // create by using super table, tag value + char *fullname; // table full name + STagData tagdata; // true tag data, super table full name is in STagData + int8_t igExist; // ignore if exists +} SCreatedTableInfo; + typedef struct SCreateTableSQL { - struct SStrToken name; // meter name, create table [meterName] xxx - bool existCheck; - - int8_t type; // create normal table/from super table/ stream + SStrToken name; // table name, create table [name] xxx + int8_t type; // create normal table/from super table/ stream + bool existCheck; + struct { - SArray *pTagColumns; // SArray - SArray *pColumns; // SArray + SArray *pTagColumns; // SArray + SArray *pColumns; // SArray } colInfo; - - struct { - SStrToken stableName; // super table name, for using clause - SArray *pTagVals; // create by using metric, tag value - STagData tagdata; - } usingInfo; - - SQuerySQL *pSelect; + + SArray *childTableInfo; // SArray + SQuerySQL *pSelect; } SCreateTableSQL; -typedef struct SAlterTableSQL { +typedef struct SAlterTableInfo { SStrToken name; + int16_t tableType; int16_t type; STagData tagData; SArray *pAddColumns; // SArray SArray *varList; // set t=val or: change src dst, SArray -} SAlterTableSQL; +} SAlterTableInfo; -typedef struct SCreateDBInfo { +typedef struct SCreateDbInfo { SStrToken dbname; int32_t replica; int32_t cacheBlockSize; @@ -116,12 +122,12 @@ typedef struct SCreateDBInfo { int32_t compressionLevel; SStrToken precision; bool ignoreExists; - int8_t update; - - SArray *keep; -} SCreateDBInfo; + int8_t update; + int8_t cachelast; + SArray *keep; +} SCreateDbInfo; -typedef struct SCreateAcctSQL { +typedef struct SCreateAcctInfo { int32_t maxUsers; int32_t maxDbs; int32_t maxTimeSeries; @@ -131,7 +137,7 @@ typedef struct SCreateAcctSQL { int64_t maxQueryTime; int32_t maxConnections; SStrToken stat; -} SCreateAcctSQL; +} SCreateAcctInfo; typedef struct SShowInfo { uint8_t showType; @@ -146,22 +152,18 @@ typedef struct SUserInfo { int16_t type; } SUserInfo; -typedef struct tDCLSQL { - int32_t nTokens; /* Number of expressions on the list */ - int32_t nAlloc; /* Number of entries allocated below */ - SStrToken *a; /* one entry for element */ - bool existsCheck; - +typedef struct SMiscInfo { + SArray *a; // SArray + bool existsCheck; + int16_t tableType; + SUserInfo user; union { - SCreateDBInfo dbOpt; - SCreateAcctSQL acctOpt; - SShowInfo showOpt; - SStrToken ip; + SCreateDbInfo dbOpt; + SCreateAcctInfo acctOpt; + SShowInfo showOpt; + SStrToken id; }; - - SUserInfo user; - -} tDCLSQL; +} SMiscInfo; typedef struct SSubclauseInfo { // "UNION" multiple select sub-clause SQuerySQL **pClause; @@ -171,15 +173,13 @@ typedef struct SSubclauseInfo { // "UNION" multiple select sub-clause typedef struct SSqlInfo { int32_t type; bool valid; - + SSubclauseInfo subclauseInfo; + char msg[256]; union { - SCreateTableSQL *pCreateTableInfo; - SAlterTableSQL *pAlterInfo; - tDCLSQL *pDCLInfo; + SCreateTableSQL *pCreateTableInfo; + SAlterTableInfo *pAlterInfo; + SMiscInfo *pMiscInfo; }; - - SSubclauseInfo subclauseInfo; - char pzErrMsg[256]; } SSqlInfo; typedef struct tSQLExpr { @@ -198,16 +198,17 @@ typedef struct tSQLExpr { } tSQLExpr; // used in select clause. select from xxx -typedef struct tSQLExprItem { +typedef struct tSqlExprItem { tSQLExpr *pNode; // The list of expressions char * aliasName; // alias name, null-terminated string -} tSQLExprItem; + bool distinct; +} tSqlExprItem; // todo refactor by using SArray typedef struct tSQLExprList { int32_t nExpr; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ - tSQLExprItem *a; /* One entry for each expression */ + tSqlExprItem *a; /* One entry for each expression */ } tSQLExprList; /** @@ -229,62 +230,63 @@ SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder); SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder); -tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType); +tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType); -void tSQLExprDestroy(tSQLExpr *); +tSQLExpr *tSqlExprClone(tSQLExpr *pSrc); -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); +void tSqlExprDestroy(tSQLExpr *pExpr); -void tSQLExprListDestroy(tSQLExprList *pList); +tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pDistinct, SStrToken *pToken); -SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, +void tSqlExprListDestroy(tSQLExprList *pList); + +SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); -SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pMetricName, - SArray *pTagVals, SQuerySQL *pSelect, int32_t type); +SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type); -void tSQLExprNodeDestroy(tSQLExpr *pExpr); +void tSqlExprNodeDestroy(tSQLExpr *pExpr); -SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type); +SAlterTableInfo * tAlterTableSqlElems(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableTable); +SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists); void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); +void freeCreateTableInfo(void* p); -SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type); +SSqlInfo * setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists); -void SQLInfoDestroy(SSqlInfo *pInfo); +void SqlInfoDestroy(SSqlInfo *pInfo); void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t tableType); void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); - -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists); -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); -void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); -void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip); -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo); +void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); +void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip); +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); -void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); // prefix show db.tables; -void setDBName(SStrToken *pCpxName, SStrToken *pDB); +void setDbName(SStrToken *pCpxName, SStrToken *pDb); -tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optType); +tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType); -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); +tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); +void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken); +void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type); void *ParseAlloc(void *(*mallocProc)(size_t)); diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 90bd64336fdeed91deb68b9b490224a7fb29bc80..5d055782c9b82a1444c97a62d429cc2ba9a53986 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -88,6 +88,7 @@ typedef struct STSBuf { STSList tsData; // uncompressed raw ts data uint64_t numOfTotal; bool autoDelete; + bool remainOpen; int32_t tsOrder; // order of timestamp in ts comp buffer STSCursor cur; } STSBuf; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index fb71c8a5fe0a7948c09c4bb4d83ed77b5d291389..d4a0c25886ad63ff5cc3e79cb0a9e84c156197cd 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -15,6 +15,8 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#include "tbuffer.h" + #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ assert(sizeof(_uid) == sizeof(uint64_t)); \ @@ -24,36 +26,31 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) + int32_t getOutputInterResultBufSize(SQuery* pQuery); -void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type); -void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); -SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); +size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv); +int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); +void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -int32_t initWindowResInfo(SResultRowInfo* pWindowResInfo, int32_t size, int16_t type); +void resetResultRowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); +int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); +void closeAllResultRows(SResultRowInfo* pResultRowInfo); -void cleanupTimeWindowInfo(SResultRowInfo* pWindowResInfo); -void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pWindowResInfo); -void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num); +int32_t initResultRow(SResultRow *pResultRow); +void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); +bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); +void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); -void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv); -int32_t numOfClosedTimeWindow(SResultRowInfo* pWindowResInfo); -void closeTimeWindow(SResultRowInfo* pWindowResInfo, int32_t slot); -void closeAllTimeWindow(SResultRowInfo* pWindowResInfo); -void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order); +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pWindowResInfo, int32_t slot) { - assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); - return pWindowResInfo->pResult[slot]; +static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { + assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); + return pResultRowInfo->pResult[slot]; } -#define curTimeWindowIndex(_winres) ((_winres)->curIndex) -#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) - -bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot); - -int32_t initResultRow(SResultRow *pResultRow); - static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); @@ -65,13 +62,10 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 pQuery->pExpr1[columnIndex].bytes * realRowId; } -bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); -bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); +bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); +bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); -__filter_func_t *getRangeFilterFuncArray(int32_t type); -__filter_func_t *getValueFilterFuncArray(int32_t type); - -size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv); +__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr); SResultRowPool* initResultRowPool(size_t size); SResultRow* getNewResultRow(SResultRowPool* p); @@ -80,8 +74,23 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); bool isPointInterpoQuery(SQuery *pQuery); +typedef struct { + SArray* pResult; // SArray + int32_t colId; +} SStddevInterResult; + +void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen); +SArray* interResFromBinary(const char* data, int32_t len); +void freeInterResult(void* param); + +void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo, int32_t offset); +void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); +bool hasRemainData(SGroupResInfo* pGroupResInfo); +bool incNextGroup(SGroupResInfo* pGroupResInfo); +int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); + +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo); #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/queryLog.h b/src/query/inc/queryLog.h index a1c447a6eb6a5f14d3b5913203a9412acce3b5c4..5c48c43c45fdb5cffdcc01901f1ede4a3b98d240 100644 --- a/src/query/inc/queryLog.h +++ b/src/query/inc/queryLog.h @@ -22,15 +22,15 @@ extern "C" { #include "tlog.h" -extern int32_t qDebugFlag; -extern int32_t tscEmbedded; +extern uint32_t qDebugFlag; -#define qFatal(...) { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", 255, __VA_ARGS__); }} -#define qError(...) { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", 255, __VA_ARGS__); }} -#define qWarn(...) { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", 255, __VA_ARGS__); }} -#define qInfo(...) { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", 255, __VA_ARGS__); }} -#define qDebug(...) { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} -#define qTrace(...) { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", 255, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", 255, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", 255, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", 255, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)a, l); }} while(0) #ifdef __cplusplus } diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index e2b3bb6cbffc21bac7462284963fda72880b9906..8a01a736b73a90ba73c5c952601cdb220765d9e6 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -36,7 +36,7 @@ %syntax_error { pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->pzErrMsg); + int32_t outputBufLen = tListLen(pInfo->msg); int32_t len = 0; if(TOKEN.z) { @@ -46,13 +46,13 @@ if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { char tmpstr[128] = {0}; memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->pzErrMsg, msg, tmpstr); + len = sprintf(pInfo->msg, msg, tmpstr); } else { - len = sprintf(pInfo->pzErrMsg, msg, &TOKEN.z[0]); + len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); } } else { - len = sprintf(pInfo->pzErrMsg, "Incomplete SQL statement"); + len = sprintf(pInfo->msg, "Incomplete SQL statement"); } assert(len <= outputBufLen); @@ -112,29 +112,35 @@ cmd ::= SHOW dbPrefix(X) STABLES. { cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } //drop configure for tables cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { X.n += Z.n; - setDropDBTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y); + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1); } -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDBTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y); } +//drop stable +cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). { + X.n += Z.n; + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, TSDB_SUPER_TABLE); +} + +cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, -1); } cmd ::= DROP DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } cmd ::= DROP USER ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } cmd ::= DROP ACCOUNT ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } @@ -149,16 +155,16 @@ cmd ::= DESCRIBE ids(X) cpxName(Y). { } /////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// -cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } -cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} +cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } +cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} -cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} -cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} +cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} +cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. @@ -168,49 +174,49 @@ ids(A) ::= ID(X). {A = X; } ids(A) ::= STRING(X). {A = X; } %type ifexists {SStrToken} -ifexists(X) ::= IF EXISTS. {X.n = 1;} -ifexists(X) ::= . {X.n = 0;} +ifexists(X) ::= IF EXISTS. { X.n = 1;} +ifexists(X) ::= . { X.n = 0;} %type ifnotexists {SStrToken} -ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} -ifnotexists(X) ::= . {X.n = 0;} +ifnotexists(X) ::= IF NOT EXISTS. { X.n = 1;} +ifnotexists(X) ::= . { X.n = 0;} /////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// //create option for dnode/db/user/account cmd ::= CREATE DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). - { setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} -cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSQL(pInfo, &X, &Y);} + { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} +cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} +cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} -pps(Y) ::= . {Y.n = 0; } -pps(Y) ::= PPS INTEGER(X). {Y = X; } +pps(Y) ::= . { Y.n = 0; } +pps(Y) ::= PPS INTEGER(X). { Y = X; } -tseries(Y) ::= . {Y.n = 0; } -tseries(Y) ::= TSERIES INTEGER(X). {Y = X; } +tseries(Y) ::= . { Y.n = 0; } +tseries(Y) ::= TSERIES INTEGER(X). { Y = X; } -dbs(Y) ::= . {Y.n = 0; } -dbs(Y) ::= DBS INTEGER(X). {Y = X; } +dbs(Y) ::= . { Y.n = 0; } +dbs(Y) ::= DBS INTEGER(X). { Y = X; } -streams(Y) ::= . {Y.n = 0; } -streams(Y) ::= STREAMS INTEGER(X). {Y = X; } +streams(Y) ::= . { Y.n = 0; } +streams(Y) ::= STREAMS INTEGER(X). { Y = X; } -storage(Y) ::= . {Y.n = 0; } -storage(Y) ::= STORAGE INTEGER(X). {Y = X; } +storage(Y) ::= . { Y.n = 0; } +storage(Y) ::= STORAGE INTEGER(X). { Y = X; } -qtime(Y) ::= . {Y.n = 0; } -qtime(Y) ::= QTIME INTEGER(X). {Y = X; } +qtime(Y) ::= . { Y.n = 0; } +qtime(Y) ::= QTIME INTEGER(X). { Y = X; } -users(Y) ::= . {Y.n = 0; } -users(Y) ::= USERS INTEGER(X). {Y = X; } +users(Y) ::= . { Y.n = 0; } +users(Y) ::= USERS INTEGER(X). { Y = X; } -conns(Y) ::= . {Y.n = 0; } -conns(Y) ::= CONNS INTEGER(X). {Y = X; } +conns(Y) ::= . { Y.n = 0; } +conns(Y) ::= CONNS INTEGER(X). { Y = X; } -state(Y) ::= . {Y.n = 0; } -state(Y) ::= STATE ids(X). {Y = X; } +state(Y) ::= . { Y.n = 0; } +state(Y) ::= STATE ids(X). { Y = X; } -%type acct_optr {SCreateAcctSQL} +%type acct_optr {SCreateAcctInfo} acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { Y.maxUsers = (K.n>0)?atoi(K.z):-1; Y.maxDbs = (E.n>0)?atoi(E.z):-1; @@ -240,8 +246,9 @@ fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; } update(Y) ::= UPDATE INTEGER(X). { Y = X; } +cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } -%type db_optr {SCreateDBInfo} +%type db_optr {SCreateDbInfo} db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} db_optr(Y) ::= db_optr(Z) cache(X). { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); } @@ -258,8 +265,9 @@ db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strto db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } -%type alter_db_optr {SCreateDBInfo} +%type alter_db_optr {SCreateDbInfo} alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } @@ -269,23 +277,31 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { X.type = 0; - tSQLSetColumnType (&A, &X); + tSqlSetColumnType (&A, &X); } //define binary type, e.g., binary(10), nchar(10) typename(A) ::= ids(X) LP signed(Y) RP. { - if (Y <= 0) { - X.type = 0; - tSQLSetColumnType(&A, &X); - } else { - X.type = -Y; // negative value of name length - tSQLSetColumnType(&A, &X); - } + if (Y <= 0) { + X.type = 0; + tSqlSetColumnType(&A, &X); + } else { + X.type = -Y; // negative value of name length + tSqlSetColumnType(&A, &X); + } +} + +// define the unsigned number type +typename(A) ::= ids(X) UNSIGNED(Z). { + X.type = 0; + X.n = ((Z.z + Z.n) - X.z); + tSqlSetColumnType (&A, &X); } %type signed {int64_t} @@ -294,36 +310,74 @@ signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} ////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE ifnotexists(Y) ids(X) cpxName(Z) create_table_args. { - X.n += Z.n; - setCreatedTableName(pInfo, &X, &Y); +cmd ::= CREATE TABLE create_table_args. {} +cmd ::= CREATE TABLE create_stable_args. {} +cmd ::= CREATE STABLE create_stable_args. {} +cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} + +%type create_table_list{SCreateTableSQL*} +%destructor create_table_list{destroyCreateTableSql($$);} +create_table_list(A) ::= create_from_stable(Z). { + SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); + pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); + + taosArrayPush(pCreateTable->childTableInfo, &Z); + pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; + A = pCreateTable; +} + +create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { + taosArrayPush(X->childTableInfo, &Z); + A = X; } %type create_table_args{SCreateTableSQL*} -create_table_args(A) ::= LP columnlist(X) RP. { - A = tSetCreateSQLElems(X, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { + A = tSetCreateSqlElems(X, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + + V.n += Z.n; + setCreatedTableName(pInfo, &V, &U); } // create super table -create_table_args(A) ::= LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateSQLElems(X, Y, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +%type create_stable_args{SCreateTableSQL*} +create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { + A = tSetCreateSqlElems(X, Y, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + + V.n += Z.n; + setCreatedTableName(pInfo, &V, &U); } // create table by using super table // create table table_name using super_table_name tags(tag_values1, tag_values2) -create_table_args(A) ::= USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. { - X.n += F.n; - A = tSetCreateSQLElems(NULL, NULL, &X, Y, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +%type create_from_stable{SCreatedTableInfo} +create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. { + X.n += F.n; + V.n += Z.n; + A = createNewChildTableInfo(&X, NULL, Y, &V, &U); +} + +create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) LP tagNamelist(P) RP TAGS LP tagitemlist(Y) RP. { + X.n += F.n; + V.n += Z.n; + A = createNewChildTableInfo(&X, P, Y, &V, &U); } +%type tagNamelist{SArray*} +%destructor tagNamelist {taosArrayDestroy($$);} +tagNamelist(A) ::= tagNamelist(X) COMMA ids(Y). {taosArrayPush(X, &Y); A = X; } +tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(A, &X);} + // create stream // create table table_name as select count(*) from super_table_name interval(time) -create_table_args(A) ::= AS select(S). { - A = tSetCreateSQLElems(NULL, NULL, NULL, NULL, S, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { + A = tSetCreateSqlElems(NULL, NULL, S, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + + V.n += Z.n; + setCreatedTableName(pInfo, &V, &U); } %type column{TAOS_FIELD} @@ -335,7 +389,7 @@ columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(T // The information used for a column is the name and type of column: // tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) column(A) ::= ids(X) typename(Y). { - tSQLSetColumnInfo(&A, &X, &Y); + tSqlSetColumnInfo(&A, &X, &Y); } %type tagitemlist {SArray*} @@ -345,10 +399,10 @@ column(A) ::= ids(X) typename(Y). { tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1); } tagitemlist(A) ::= tagitem(X). { A = tVariantListAppend(NULL, &X, -1); } -tagitem(A) ::= INTEGER(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } -tagitem(A) ::= FLOAT(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } -tagitem(A) ::= STRING(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } -tagitem(A) ::= BOOL(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= INTEGER(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= STRING(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= BOOL(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= NULL(X). { X.type = 0; tVariantCreate(&A, &X); } tagitem(A) ::= MINUS(X) INTEGER(Y).{ @@ -383,7 +437,7 @@ tagitem(A) ::= PLUS(X) FLOAT(Y). { %type select {SQuerySQL*} %destructor select {doDestroyQuerySql($$);} select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { - A = tSetQuerySQLElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); + A = tSetQuerySqlElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); } %type union {SSubclauseInfo*} @@ -394,33 +448,33 @@ union(Y) ::= LP union(X) RP. { Y = X; } union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); } union(Y) ::= union(Z) UNION ALL LP select(X) RP. { Y = appendSelectClause(Z, X); } -cmd ::= union(X). { setSQLInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } +cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } // Support for the SQL exprssion without from & where subclauses, e.g., // select current_database(), // select server_version(), select client_version(), // select server_state(); select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySQLElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + A = tSetQuerySqlElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } // selcollist is a list of expressions that are to become the return // values of the SELECT statement. The "*" in statements like // "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. %type selcollist {tSQLExprList*} -%destructor selcollist {tSQLExprListDestroy($$);} +%destructor selcollist {tSqlExprListDestroy($$);} %type sclp {tSQLExprList*} -%destructor sclp {tSQLExprListDestroy($$);} +%destructor sclp {tSqlExprListDestroy($$);} sclp(A) ::= selcollist(X) COMMA. {A = X;} sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) expr(X) as(Y). { - A = tSQLExprListAppend(P, X, Y.n?&Y:0); +selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). { + A = tSqlExprListAppend(P, X, Z.n? &Z:0, Y.n?&Y:0); } selcollist(A) ::= sclp(P) STAR. { - tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - A = tSQLExprListAppend(P, pNode, 0); + tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); + A = tSqlExprListAppend(P, pNode, 0, 0); } // An option "AS " phrase that can follow one of the expressions that @@ -431,6 +485,10 @@ as(X) ::= AS ids(Y). { X = Y; } as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } +%type distinct {SStrToken} +distinct(X) ::= DISTINCT(Y). { X = Y; } +distinct(X) ::= . { X.n = 0;} + // A complete FROM clause. %type from {SArray*} // current not support query from no-table @@ -445,11 +503,11 @@ tablelist(A) ::= ids(X) cpxName(Y). { } tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { - toTSDBType(X.type); - toTSDBType(Z.type); - X.n += Y.n; - A = tVariantListAppendToken(NULL, &X, -1); - A = tVariantListAppendToken(A, &Z, -1); + toTSDBType(X.type); + toTSDBType(Z.type); + X.n += Y.n; + A = tVariantListAppendToken(NULL, &X, -1); + A = tVariantListAppendToken(A, &Z, -1); } tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { @@ -460,11 +518,11 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { } tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { - toTSDBType(X.type); - toTSDBType(F.type); - X.n += Z.n; - A = tVariantListAppendToken(Y, &X, -1); - A = tVariantListAppendToken(A, &F, -1); + toTSDBType(X.type); + toTSDBType(F.type); + X.n += Z.n; + A = tVariantListAppendToken(Y, &X, -1); + A = tVariantListAppendToken(A, &F, -1); } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" @@ -526,9 +584,9 @@ item(A) ::= ids(X) cpxName(Y). { } %type sortorder {int} -sortorder(A) ::= ASC. {A = TSDB_ORDER_ASC; } -sortorder(A) ::= DESC. {A = TSDB_ORDER_DESC;} -sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend order +sortorder(A) ::= ASC. { A = TSDB_ORDER_ASC; } +sortorder(A) ::= DESC. { A = TSDB_ORDER_DESC;} +sortorder(A) ::= . { A = TSDB_ORDER_ASC; } // Ascending order by default //group by clause %type groupby_opt {SArray*} @@ -536,8 +594,8 @@ sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend orde %type grouplist {SArray*} %destructor grouplist {taosArrayDestroy($$);} -groupby_opt(A) ::= . {A = 0;} -groupby_opt(A) ::= GROUP BY grouplist(X). {A = X;} +groupby_opt(A) ::= . { A = 0;} +groupby_opt(A) ::= GROUP BY grouplist(X). { A = X;} grouplist(A) ::= grouplist(X) COMMA item(Y). { A = tVariantListAppend(X, &Y, -1); @@ -549,7 +607,7 @@ grouplist(A) ::= item(X). { //having clause, ignore the input condition in having %type having_opt {tSQLExpr*} -%destructor having_opt {tSQLExprDestroy($$);} +%destructor having_opt {tSqlExprDestroy($$);} having_opt(A) ::=. {A = 0;} having_opt(A) ::= HAVING expr(X). {A = X;} @@ -571,7 +629,7 @@ slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). {A.limit = Y; A.offset = X;} %type where_opt {tSQLExpr*} -%destructor where_opt {tSQLExprDestroy($$);} +%destructor where_opt {tSqlExprDestroy($$);} where_opt(A) ::= . {A = 0;} where_opt(A) ::= WHERE expr(X). {A = X;} @@ -579,67 +637,69 @@ where_opt(A) ::= WHERE expr(X). {A = X;} /////////////////////////// Expression Processing ///////////////////////////// // %type expr {tSQLExpr*} -%destructor expr {tSQLExprDestroy($$);} +%destructor expr {tSqlExprDestroy($$);} expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->token.z = X.z; A->token.n = (Z.z - X.z + 1);} -expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);} -expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);} +expr(A) ::= ID(X). { A = tSqlExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ALL);} + +expr(A) ::= INTEGER(X). { A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= FLOAT(X). { A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= STRING(X). { A = tSqlExprIdValueCreate(&X, TK_STRING);} +expr(A) ::= NOW(X). { A = tSqlExprIdValueCreate(&X, TK_NOW); } +expr(A) ::= VARIABLE(X). { A = tSqlExprIdValueCreate(&X, TK_VARIABLE);} +expr(A) ::= BOOL(X). { A = tSqlExprIdValueCreate(&X, TK_BOOL);} // ordinary functions: min(x), max(x), top(k, 20) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSQLExprCreateFunction(Y, &X, &E, X.type); } +expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); } // for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); } +expr(A) ::= ID(X) LP STAR RP(Y). { A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); } // is (not) null expression -expr(A) ::= expr(X) IS NULL. {A = tSQLExprCreate(X, NULL, TK_ISNULL);} -expr(A) ::= expr(X) IS NOT NULL. {A = tSQLExprCreate(X, NULL, TK_NOTNULL);} +expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);} +expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);} // relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);} +expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);} +expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);} +expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);} +expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} +expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} +expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} + +expr(A) ::= expr(X) BETWEEN expr(Y) AND expr(Z). { tSQLExpr* X2 = tSqlExprClone(X); A = tSqlExprCreate(tSqlExprCreate(X, Y, TK_GE), tSqlExprCreate(X2, Z, TK_LE), TK_AND);} -expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); } +expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} +expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } // binary arithmetic expression -expr(A) ::= expr(X) PLUS expr(Y). {A = tSQLExprCreate(X, Y, TK_PLUS); } -expr(A) ::= expr(X) MINUS expr(Y). {A = tSQLExprCreate(X, Y, TK_MINUS); } -expr(A) ::= expr(X) STAR expr(Y). {A = tSQLExprCreate(X, Y, TK_STAR); } -expr(A) ::= expr(X) SLASH expr(Y). {A = tSQLExprCreate(X, Y, TK_DIVIDE);} -expr(A) ::= expr(X) REM expr(Y). {A = tSQLExprCreate(X, Y, TK_REM); } +expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); } +expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); } +expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); } +expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);} +expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } // like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); } +expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } //in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, TK_IN); } +expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSQLExpr*)Y, TK_IN); } %type exprlist {tSQLExprList*} -%destructor exprlist {tSQLExprListDestroy($$);} +%destructor exprlist {tSqlExprListDestroy($$);} %type expritem {tSQLExpr*} -%destructor expritem {tSQLExprDestroy($$);} +%destructor expritem {tSqlExprDestroy($$);} -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSQLExprListAppend(X,Y,0);} -exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);} +exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);} +exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0, 0);} expritem(A) ::= expr(X). {A = X;} expritem(A) ::= . {A = 0;} @@ -649,8 +709,8 @@ cmd ::= RESET QUERY CACHE. { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} ///////////////////////////////////ALTER TABLE statement////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { X.n += F.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { @@ -659,15 +719,15 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { toTSDBType(A.type); SArray* K = tVariantListAppendToken(NULL, &A, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } //////////////////////////////////ALTER TAGS statement///////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { X.n += Y.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { X.n += Z.n; @@ -675,8 +735,8 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { toTSDBType(Y.type); SArray* A = tVariantListAppendToken(NULL, &Y, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { @@ -688,8 +748,8 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { toTSDBType(Z.type); A = tVariantListAppendToken(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { @@ -699,14 +759,61 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { SArray* A = tVariantListAppendToken(NULL, &Y, -1); A = tVariantListAppend(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + + +///////////////////////////////////ALTER STABLE statement////////////////////////////////// +cmd ::= ALTER STABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { + X.n += F.n; + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + +cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). { + X.n += F.n; + + toTSDBType(A.type); + SArray* K = tVariantListAppendToken(NULL, &A, -1); + + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + +//////////////////////////////////ALTER TAGS statement///////////////////////////////////// +cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { + X.n += Y.n; + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} +cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { + X.n += Z.n; + + toTSDBType(Y.type); + SArray* A = tVariantListAppendToken(NULL, &Y, -1); + + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + +cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { + X.n += F.n; + + toTSDBType(Y.type); + SArray* A = tVariantListAppendToken(NULL, &Y, -1); + + toTSDBType(Z.type); + A = tVariantListAppendToken(A, &Z, -1); + + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } ////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION INTEGER(Y). {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} -cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &X);} +cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} +cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);} +cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);} %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD diff --git a/src/client/src/tscFunctionImpl.c b/src/query/src/qAggMain.c similarity index 72% rename from src/client/src/tscFunctionImpl.c rename to src/query/src/qAggMain.c index 7921399330f876a217b8f14f4871d7f47cebec21..f99fe3f0720784db0a5e0b526dddb436a91f87d3 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/query/src/qAggMain.c @@ -14,28 +14,29 @@ */ #include "os.h" -#include "qAst.h" -#include "qExtbuffer.h" +#include "taosdef.h" +#include "taosmsg.h" +#include "texpr.h" +#include "ttype.h" + +#include "qAggMain.h" #include "qFill.h" #include "qHistogram.h" #include "qPercentile.h" -#include "qSyntaxtreefunction.h" #include "qTsbuf.h" -#include "taosdef.h" -#include "taosmsg.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscompression.h" -#include "tsqlfunction.h" -#include "tutil.h" -#include "ttype.h" +#include "queryLog.h" + +//#define GET_INPUT_DATA_LIST(x) (((char *)((x)->pInput)) + ((x)->startOffset) * ((x)->inputBytes)) +#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) +#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) -#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) -#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes) +//#define GET_TS_LIST(x) ((TSKEY*)&((x)->ptsList[(x)->startOffset])) +#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) #define GET_TRUE_DATA_TYPE() \ int32_t type = 0; \ - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { \ + if (pCtx->currentStage == MERGE_STAGE) { \ type = pCtx->outputType; \ assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ } else { \ @@ -48,21 +49,21 @@ break; \ } \ GET_RES_INFO(ctx)->numOfRes = (res); \ - } while (0); + } while (0) #define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); -#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ - __ctx->tag.i64Key = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ - } \ - } while (0); +#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ + __ctx->tag.i64 = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ + } \ + } while (0) #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ @@ -91,8 +92,9 @@ typedef struct SSpreadInfo { typedef struct SSumInfo { union { - int64_t isum; - double dsum; + int64_t isum; + uint64_t usum; + double dsum; }; int8_t hasResult; } SSumInfo; @@ -110,6 +112,11 @@ typedef struct SStddevInfo { int8_t stage; } SStddevInfo; +typedef struct SStddevdstInfo { + int64_t num; + double res; +} SStddevdstInfo; + typedef struct SFirstLastInfo { int8_t hasResult; TSKEY ts; @@ -159,7 +166,7 @@ typedef struct SRateInfo { int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) { if (!isValidDataType(dataType)) { - tscError("Illegal data type %d or data type length %d", dataType, dataBytes); + qError("Illegal data type %d or data type length %d", dataType, dataBytes); return TSDB_CODE_TSC_INVALID_SQL; } @@ -264,8 +271,10 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } if (functionId == TSDB_FUNC_SUM) { - if (dataType >= TSDB_DATA_TYPE_TINYINT && dataType <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(dataType)) { *type = TSDB_DATA_TYPE_BIGINT; + } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { + *type = TSDB_DATA_TYPE_UBIGINT; } else { *type = TSDB_DATA_TYPE_DOUBLE; } @@ -334,6 +343,11 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; *interBytes = dataBytes; + } else if (functionId == TSDB_FUNC_STDDEV_DST) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(SStddevdstInfo); + *interBytes = (*bytes); + } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -353,7 +367,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { return false; } - memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); + memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); initResultInfo(pResInfo, pCtx->interBufBytes); return true; } @@ -368,11 +382,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { static void function_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->outputType); - } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - } + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } doFinalizer(pCtx); @@ -395,7 +405,7 @@ static void count_function(SQLFunctionCtx *pCtx) { } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_CHAR_INDEX(pCtx, i); + char *val = GET_INPUT_DATA(pCtx, i); if (isNull(val, pCtx->inputType)) { continue; } @@ -403,10 +413,7 @@ static void count_function(SQLFunctionCtx *pCtx) { numOfElem += 1; } } else { - /* - * when counting on the primary time stamp column and no statistics data is provided, - * simple use the size value - */ + //when counting on the primary time stamp column and no statistics data is presented, use the size value directly. numOfElem = pCtx->size; } } @@ -415,18 +422,18 @@ static void count_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } - *((int64_t *)pCtx->aOutputBuf) += numOfElem; + *((int64_t *)pCtx->pOutput) += numOfElem; SET_VAL(pCtx, numOfElem, 1); } static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } SET_VAL(pCtx, 1, 1); - *((int64_t *)pCtx->aOutputBuf) += pCtx->size; + *((int64_t *)pCtx->pOutput) += pCtx->size; // do not need it actually SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -434,9 +441,9 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void count_func_merge(SQLFunctionCtx *pCtx) { - int64_t *pData = (int64_t *)GET_INPUT_CHAR(pCtx); + int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { - *((int64_t *)pCtx->aOutputBuf) += pData[i]; + *((int64_t *)pCtx->pOutput) += pData[i]; } SET_VAL(pCtx, pCtx->size, 1); @@ -464,7 +471,7 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId } #define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType) \ - { \ + do { \ t *d = (t *)(p); \ for (int32_t i = 0; i < (ctx)->size; ++i) { \ if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ @@ -473,7 +480,7 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId (x) += (d)[i]; \ (numOfElem)++; \ } \ - }; + } while(0) #define UPDATE_DATA(ctx, left, right, num, sign, k) \ do { \ @@ -482,24 +489,23 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId DO_UPDATE_TAG_COLUMNS(ctx, k); \ (num) += 1; \ } \ - } while (0); + } while (0) #define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ -do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ - (num) += 1; \ - } \ - } while (0); - + (num) += 1; \ + } \ + } while (0) #define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ for (int32_t i = 0; i < ((ctx)->size); ++i) { \ if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ continue; \ } \ - TSKEY key = (ctx)->ptsList[i]; \ + TSKEY key = GET_TS_DATA(ctx, i); \ UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ } @@ -518,20 +524,23 @@ static void do_sum(SQLFunctionCtx *pCtx) { notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull); - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = (int64_t*) pCtx->aOutputBuf; + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { + int64_t *retVal = (int64_t *)pCtx->pOutput; *retVal += pCtx->preAggVals.statis.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double*) pCtx->aOutputBuf; + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + uint64_t *retVal = (uint64_t *)pCtx->pOutput; + *retVal += (uint64_t)pCtx->preAggVals.statis.sum; + } else if (IS_FLOAT_TYPE(pCtx->inputType)) { + double *retVal = (double*) pCtx->pOutput; *retVal += GET_DOUBLE_VAL((const char*)&(pCtx->preAggVals.statis.sum)); } } else { // computing based on the true data block - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); notNullElems = 0; - - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = (int64_t*) pCtx->aOutputBuf; - + + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { + int64_t *retVal = (int64_t *)pCtx->pOutput; + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -541,11 +550,23 @@ static void do_sum(SQLFunctionCtx *pCtx) { } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); } + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + uint64_t *retVal = (uint64_t *)pCtx->pOutput; + + if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + LIST_ADD_N(*retVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + LIST_ADD_N(*retVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + LIST_ADD_N(*retVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + LIST_ADD_N(*retVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); + } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double*) pCtx->aOutputBuf; + double *retVal = (double *)pCtx->pOutput; LIST_ADD_N(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double*) pCtx->aOutputBuf; + double *retVal = (double *)pCtx->pOutput; LIST_ADD_N(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } @@ -559,13 +580,13 @@ static void do_sum(SQLFunctionCtx *pCtx) { } static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } SET_VAL(pCtx, 1, 1); - int64_t *res = (int64_t*) pCtx->aOutputBuf; + int64_t *res = (int64_t*) pCtx->pOutput; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { *res += GET_INT8_VAL(pData); @@ -575,11 +596,23 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { *res += GET_INT32_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { *res += GET_INT64_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT8_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT16_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT32_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT64_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double*) pCtx->aOutputBuf; + double *retVal = (double*) pCtx->pOutput; *retVal += GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double*) pCtx->aOutputBuf; + double *retVal = (double*) pCtx->pOutput; *retVal += GET_FLOAT_VAL(pData); } @@ -593,7 +626,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { // set the flag for super table query - SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; + SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; pSum->hasResult = DATA_SET_FLAG; } } @@ -604,59 +637,35 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the result data in output buffer, not in the intermediate buffer SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; + SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; pSum->hasResult = DATA_SET_FLAG; } } -static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { +static void sum_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); assert(pCtx->stableQuery); - + for (int32_t i = 0; i < pCtx->size; ++i) { - char * input = GET_INPUT_CHAR_INDEX(pCtx, i); + char * input = GET_INPUT_DATA(pCtx, i); SSumInfo *pInput = (SSumInfo *)input; if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + notNullElems++; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: { - *(int64_t *)pCtx->aOutputBuf += pInput->isum; - break; - }; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)pCtx->aOutputBuf += pInput->dsum; - } - } - } - - return notNullElems; -} -static void sum_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - - SET_VAL(pCtx, notNullElems, 1); - SSumInfo *pSumInfo = (SSumInfo *)pCtx->aOutputBuf; - - if (notNullElems > 0) { - // pCtx->numOfIteratedElems += notNullElems; - pSumInfo->hasResult = DATA_SET_FLAG; + if (IS_SIGNED_NUMERIC_TYPE(type)) { + *(int64_t *)pCtx->pOutput += pInput->isum; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + *(uint64_t *) pCtx->pOutput += pInput->usum; + } else { + *(double *)pCtx->pOutput += pInput->dsum; + } } -} -static void sum_func_second_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - SET_VAL(pCtx, notNullElems, 1); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -688,7 +697,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, i } static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order != pCtx->param[0].i64Key) { + if (pCtx->order != pCtx->param[0].i64) { return BLK_DATA_NO_NEEDED; } @@ -705,41 +714,38 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY en } // not initialized yet, it is the first block, load it. - if (pCtx->aOutputBuf == NULL) { + if (pCtx->pOutput == NULL) { return BLK_DATA_ALL_NEEDED; } - return BLK_DATA_ALL_NEEDED; - // TODO pCtx->aOutputBuf is the previous windowRes output buffer, not current unloaded block. so the following filter - // is invalid -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { // data in current block is not earlier than current result -// return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is + // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { // data in current block is not earlier than current result + return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order != pCtx->param[0].i64Key) { + if (pCtx->order != pCtx->param[0].i64) { return BLK_DATA_NO_NEEDED; } // not initialized yet, it is the first block, load it. - if (pCtx->aOutputBuf == NULL) { + if (pCtx->pOutput == NULL) { return BLK_DATA_ALL_NEEDED; } - return BLK_DATA_ALL_NEEDED; - // TODO pCtx->aOutputBuf is the previous windowRes output buffer, not current unloaded block. so the following filter - // is invalid - -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { -// return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is + // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { + return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } ////////////////////////////////////////////////////////////////////////////////////////////// @@ -755,20 +761,21 @@ static void avg_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - double * pVal = &pAvgInfo->sum; + double *pVal = &pAvgInfo->sum; - if (pCtx->preAggVals.isSet) { - // Pre-aggregation + if (pCtx->preAggVals.isSet) { // Pre-aggregation notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(notNullElems >= 0); - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { *pVal += pCtx->preAggVals.statis.sum; + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + *pVal += (uint64_t) pCtx->preAggVals.statis.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.sum)); } } else { - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); @@ -782,6 +789,14 @@ static void avg_function(SQLFunctionCtx *pCtx) { LIST_ADD_N(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { LIST_ADD_N(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + LIST_ADD_N(*pVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + LIST_ADD_N(*pVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + LIST_ADD_N(*pVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); } } @@ -798,12 +813,12 @@ static void avg_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -827,8 +842,16 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { pAvgInfo->sum += GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { pAvgInfo->sum += GET_FLOAT_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + pAvgInfo->sum += GET_UINT8_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + pAvgInfo->sum += GET_UINT16_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + pAvgInfo->sum += GET_UINT32_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + pAvgInfo->sum += GET_UINT64_VAL(pData); } - + // restore sum and count of elements pAvgInfo->num += 1; @@ -837,39 +860,15 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current buffer is null - continue; - } - - pAvgInfo->sum += pInput->sum; - pAvgInfo->num += pInput->num; - } - - // if the data set hasResult is not set, the result is null - if (pAvgInfo->num > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_second_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - double *sum = (double*) pCtx->aOutputBuf; - char * input = GET_INPUT_CHAR(pCtx); + double *sum = (double*) pCtx->pOutput; + char *input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; @@ -890,26 +889,25 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { static void avg_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - return; // empty table + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + return; } - *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo); + *(double *)pCtx->pOutput = (*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo); } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY - assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE); - + assert(IS_NUMERIC_TYPE(pCtx->inputType)); SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } - *(double *)pCtx->aOutputBuf = pAvgInfo->sum / pAvgInfo->num; + *(double *)pCtx->pOutput = pAvgInfo->sum / pAvgInfo->num; } // cannot set the numOfIteratedElems again since it is set during previous iteration @@ -924,8 +922,12 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (pCtx->preAggVals.isSet) { *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(*notNullElems >= 0); - - void * tval = NULL; + + if (*notNullElems == 0) { + return; + } + + void* tval = NULL; int16_t index = 0; if (isMin) { @@ -944,14 +946,15 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, * * The following codes of 3 lines will be removed later. */ - if (index < 0 || index >= pCtx->size + pCtx->startOffset) { - index = 0; - } - +// if (index < 0 || index >= pCtx->size + pCtx->startOffset) { +// index = 0; +// } + + // the index is the original position, not the relative position key = pCtx->ptsList[index]; } - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { int64_t val = GET_INT64_VAL(tval); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *data = (int8_t *)pOutput; @@ -964,7 +967,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *data = (int32_t *)pOutput; #if defined(_DEBUG_VIEW) - tscDebug("max value updated according to pre-cal:%d", *data); + qDebug("max value updated according to pre-cal:%d", *data); #endif if ((*data < val) ^ isMin) { @@ -972,7 +975,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { - __ctx->tag.i64Key = key; + __ctx->tag.i64 = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; } @@ -983,11 +986,27 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int64_t *data = (int64_t *)pOutput; UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + uint64_t val = GET_UINT64_VAL(tval); + if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + uint8_t *data = (uint8_t *)pOutput; + + UPDATE_DATA(pCtx, *data, (uint8_t)val, notNullElems, isMin, key); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + uint16_t *data = (uint16_t *)pOutput; + UPDATE_DATA(pCtx, *data, (uint16_t)val, notNullElems, isMin, key); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + uint32_t *data = (uint32_t *)pOutput; + UPDATE_DATA(pCtx, *data, (uint32_t)val, notNullElems, isMin, key); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + uint64_t *data = (uint64_t *)pOutput; + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); + } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *data = (double *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); + double *data = (double *)pOutput; + double val = GET_DOUBLE_VAL(tval); + + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *data = (float *)pOutput; double val = GET_DOUBLE_VAL(tval); @@ -998,10 +1017,12 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, return; } - void *p = GET_INPUT_CHAR(pCtx); + void *p = GET_INPUT_DATA_LIST(pCtx); + TSKEY *tsList = GET_TS_LIST(pCtx); + *notNullElems = 0; - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -1017,7 +1038,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; - TSKEY k = pCtx->ptsList[i]; + TSKEY k = tsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); } @@ -1025,11 +1046,21 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, *notNullElems += 1; } #if defined(_DEBUG_VIEW) - tscDebug("max value updated:%d", *retVal); + qDebug("max value updated:%d", *retVal); #endif } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); } + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + TYPED_LOOPCHECK_N(uint8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + TYPED_LOOPCHECK_N(uint16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + TYPED_LOOPCHECK_N(uint32_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + TYPED_LOOPCHECK_N(uint64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { TYPED_LOOPCHECK_N(double, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { @@ -1045,28 +1076,40 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) { GET_TRUE_DATA_TYPE(); switch (type) { - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->aOutputBuf) = INT32_MAX; + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)pCtx->pOutput) = INT8_MAX; break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->aOutputBuf) = FLT_MAX; + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *) pCtx->pOutput = UINT8_MAX; break; - case TSDB_DATA_TYPE_DOUBLE: - *((double *)pCtx->aOutputBuf) = DBL_MAX; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)pCtx->pOutput) = INT16_MAX; + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)pCtx->pOutput) = UINT16_MAX; + break; + case TSDB_DATA_TYPE_INT: + *((int32_t *)pCtx->pOutput) = INT32_MAX; + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)pCtx->pOutput) = UINT32_MAX; break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->aOutputBuf) = INT64_MAX; + *((int64_t *)pCtx->pOutput) = INT64_MAX; break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->aOutputBuf) = INT16_MAX; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)pCtx->pOutput) = UINT64_MAX; break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->aOutputBuf) = INT8_MAX; + case TSDB_DATA_TYPE_FLOAT: + *((float *)pCtx->pOutput) = FLT_MAX; + break; + case TSDB_DATA_TYPE_DOUBLE: + *((double *)pCtx->pOutput) = DBL_MAX; break; default: - tscError("illegal data type:%d in min/max query", pCtx->inputType); + qError("illegal data type:%d in min/max query", pCtx->inputType); } - + return true; } @@ -1079,25 +1122,37 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { switch (type) { case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->aOutputBuf) = INT32_MIN; + *((int32_t *)pCtx->pOutput) = INT32_MIN; + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)pCtx->pOutput) = 0; break; case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->aOutputBuf) = -FLT_MAX; + *((float *)pCtx->pOutput) = -FLT_MAX; break; case TSDB_DATA_TYPE_DOUBLE: - *((double *)pCtx->aOutputBuf) = -DBL_MAX; + *((double *)pCtx->pOutput) = -DBL_MAX; break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->aOutputBuf) = INT64_MIN; + *((int64_t *)pCtx->pOutput) = INT64_MIN; + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)pCtx->pOutput) = 0; break; case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->aOutputBuf) = INT16_MIN; + *((int16_t *)pCtx->pOutput) = INT16_MIN; + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)pCtx->pOutput) = 0; break; case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->aOutputBuf) = INT8_MIN; + *((int8_t *)pCtx->pOutput) = INT8_MIN; + break; + case TSDB_DATA_TYPE_UTINYINT: + *((uint8_t *)pCtx->pOutput) = 0; break; default: - tscError("illegal data type:%d in min/max query", pCtx->inputType); + qError("illegal data type:%d in min/max query", pCtx->inputType); } return true; @@ -1108,7 +1163,7 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { */ static void min_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->aOutputBuf, 1, ¬NullElems); + minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); SET_VAL(pCtx, notNullElems, 1); @@ -1118,14 +1173,14 @@ static void min_function(SQLFunctionCtx *pCtx) { // set the flag for super table query if (pCtx->stableQuery) { - *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; + *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; } } } static void max_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->aOutputBuf, 0, ¬NullElems); + minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); SET_VAL(pCtx, notNullElems, 1); @@ -1135,7 +1190,7 @@ static void max_function(SQLFunctionCtx *pCtx) { // set the flag for super table query if (pCtx->stableQuery) { - *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; + *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; } } } @@ -1144,11 +1199,10 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_CHAR_INDEX(pCtx, i); + char *input = GET_INPUT_DATA(pCtx, i); if (input[bytes] != DATA_SET_FLAG) { continue; } @@ -1158,7 +1212,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp int8_t v = GET_INT8_VAL(input); DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin); break; - }; + } case TSDB_DATA_TYPE_SMALLINT: { int16_t v = GET_INT16_VAL(input); DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin); @@ -1192,7 +1246,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp int64_t v = GET_INT64_VAL(input); DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin); break; - }; + } default: break; } @@ -1202,18 +1256,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp } static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void min_func_second_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); + int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1224,17 +1267,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { } static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); - - SET_VAL(pCtx, numOfElems, 1); - if (numOfElems > 0) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void max_func_second_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); + int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1245,37 +1278,37 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { } static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - TSKEY key = pCtx->ptsList[index]; + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY key = GET_TS_DATA(pCtx, index); int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *output = (int8_t *)pCtx->aOutputBuf; + int8_t *output = (int8_t *)pCtx->pOutput; int8_t i = GET_INT8_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *output = (int16_t*) pCtx->aOutputBuf; + int16_t *output = (int16_t*) pCtx->pOutput; int16_t i = GET_INT16_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *output = (int32_t*) pCtx->aOutputBuf; + int32_t *output = (int32_t*) pCtx->pOutput; int32_t i = GET_INT32_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *output = (int64_t*) pCtx->aOutputBuf; + int64_t *output = (int64_t*) pCtx->pOutput; int64_t i = GET_INT64_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *output = (float*) pCtx->aOutputBuf; + float *output = (float*) pCtx->pOutput; float i = GET_FLOAT_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *output = (double*) pCtx->aOutputBuf; + double *output = (double*) pCtx->pOutput; double i = GET_DOUBLE_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); @@ -1285,7 +1318,7 @@ static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin } static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1295,13 +1328,13 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + char *flag = pCtx->pOutput + pCtx->inputBytes; *flag = DATA_SET_FLAG; } } static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1311,66 +1344,86 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + char *flag = pCtx->pOutput + pCtx->inputBytes; *flag = DATA_SET_FLAG; } } -#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, tsdbType) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], tsdbType)) { \ - continue; \ - } \ - (r) += POW2(((type *)d)[i] - (delta)); \ +#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ + for (int32_t i = 0; i < (ctx)->size; ++i) { \ + if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ + continue; \ + } \ + (num) += 1; \ + (r) += POW2(((type *)d)[i] - (delta)); \ } static void stddev_function(SQLFunctionCtx *pCtx) { - // the second stage to calculate standard deviation SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - if (pStd->stage == 0) { // the first stage is to calculate average value + if (pStd->stage == 0) { + // the first stage is to calculate average value avg_function(pCtx); - } else { + } else if (pStd->num > 0) { + // the second stage to calculate standard deviation + // if pStd->num == 0, there are no numbers in the first round check. No need to do the second round double *retVal = &pStd->res; double avg = pStd->avg; - void *pData = GET_INPUT_CHAR(pCtx); - + void *pData = GET_INPUT_DATA_LIST(pCtx); + int32_t num = 0; + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pCtx->size; ++i) { if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { continue; } + num += 1; *retVal += POW2(((int32_t *)pData)[i] - avg); } break; } case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UINT: { + LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } default: - tscError("stddev function not support data type:%d", pCtx->inputType); + qError("stddev function not support data type:%d", pCtx->inputType); } - // TODO get the correct data SET_VAL(pCtx, 1, 1); } } @@ -1383,9 +1436,9 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { /* the first stage is to calculate average value */ if (pStd->stage == 0) { avg_function_f(pCtx, index); - } else { + } else if (pStd->num > 0) { double avg = pStd->avg; - void * pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void * pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; @@ -1416,8 +1469,24 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { pStd->res += POW2(GET_INT8_VAL(pData) - avg); break; } + case TSDB_DATA_TYPE_UINT: { + pStd->res += POW2(GET_UINT32_VAL(pData) - avg); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + pStd->res += POW2(GET_UINT64_VAL(pData) - avg); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + pStd->res += POW2(GET_UINT16_VAL(pData) - avg); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + pStd->res += POW2(GET_UINT8_VAL(pData) - avg); + break; + } default: - tscError("stddev function not support data type:%d", pCtx->inputType); + qError("stddev function not support data type:%d", pCtx->inputType); } SET_VAL(pCtx, 1, 1); @@ -1447,7 +1516,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { // save average value into tmpBuf, for second stage scan SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); + pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); } else { pResInfo->complete = true; @@ -1458,9 +1527,9 @@ static void stddev_finalizer(SQLFunctionCtx *pCtx) { SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } else { - double *retValue = (double *)pCtx->aOutputBuf; + double *retValue = (double *)pCtx->pOutput; *retValue = sqrt(pStd->res / pStd->num); SET_VAL(pCtx, 1, 1); } @@ -1469,63 +1538,287 @@ static void stddev_finalizer(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx) { - if (!function_setup(pCtx)) { - return false; +int32_t tsCompare(const void* p1, const void* p2) { + TSKEY k = *(TSKEY*)p1; + SResPair* pair = (SResPair*)p2; + + if (k == pair->key) { + return 0; + } else { + return k < pair->key? -1:1; } - - // used to keep the timestamp for comparison - pCtx->param[1].nType = 0; - pCtx->param[1].i64Key = 0; - - return true; } -// todo opt for null block -static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) { +static void stddev_dst_function(SQLFunctionCtx *pCtx) { + SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + // the second stage to calculate standard deviation + double *retVal = &pStd->res; + + // all data are null, no need to proceed + SArray* resList = (SArray*) pCtx->param[0].pz; + if (resList == NULL) { return; } - - int32_t notNullElems = 0; - - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; + + // find the correct group average results according to the tag value + int32_t len = (int32_t) taosArrayGetSize(resList); + assert(len > 0); + + double avg = 0; + if (len == 1) { + SResPair* p = taosArrayGet(resList, 0); + avg = p->avg; + } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result + SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); + if (p == NULL) { + return; } - memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - - TSKEY k = pCtx->ptsList[i]; - DO_UPDATE_TAG_COLUMNS(pCtx, k); - - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; - - notNullElems++; - break; + avg = p->avg; } - - SET_VAL(pCtx, notNullElems, 1); + + void *pData = GET_INPUT_DATA_LIST(pCtx); + int32_t num = 0; + + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_INT: { + for (int32_t i = 0; i < pCtx->size; ++i) { + if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { + continue; + } + num += 1; + *retVal += POW2(((int32_t *)pData)[i] - avg); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UINT: { + LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + default: + qError("stddev function not support data type:%d", pCtx->inputType); + } + + pStd->num += num; + SET_VAL(pCtx, num, 1); + + // copy to the final output buffer for super table + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); } -static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->order == TSDB_ORDER_DESC) { +static void stddev_dst_function_f(SQLFunctionCtx *pCtx, int32_t index) { + void *pData = GET_INPUT_DATA(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + + // the second stage to calculate standard deviation + SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + double *retVal = &pStd->res; + + // all data are null, no need to proceed + SArray* resList = (SArray*) pCtx->param[0].pz; + if (resList == NULL) { + return; + } + + // find the correct group average results according to the tag value + int32_t len = (int32_t) taosArrayGetSize(resList); + assert(len > 0); + + double avg = 0; + if (len == 1) { + SResPair* p = taosArrayGet(resList, 0); + avg = p->avg; + } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result + SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); + assert(p != NULL); + + avg = p->avg; + } + + int32_t num = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_INT: { + for (int32_t i = 0; i < pCtx->size; ++i) { + if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { + continue; + } + num += 1; + *retVal += POW2(((int32_t *)pData)[i] - avg); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UINT: { + LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + default: + qError("stddev function not support data type:%d", pCtx->inputType); + } + + pStd->num += num; + SET_VAL(pCtx, num, 1); + + // copy to the final output buffer for super table + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); +} + + +static void stddev_dst_merge(SQLFunctionCtx *pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); + + char *input = GET_INPUT_DATA_LIST(pCtx); + + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { + SStddevdstInfo *pInput = (SStddevdstInfo *)input; + if (pInput->num == 0) { // current input is null + continue; + } + + pRes->num += pInput->num; + pRes->res += pInput->res; + } +} + +static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { + SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + if (pStd->num <= 0) { + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + } else { + double *retValue = (double *)pCtx->pOutput; + *retValue = sqrt(pStd->res / pStd->num); + SET_VAL(pCtx, 1, 1); + } + + doFinalizer(pCtx); +} + +////////////////////////////////////////////////////////////////////////////////////// +static bool first_last_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; + } + + // used to keep the timestamp for comparison + pCtx->param[1].nType = 0; + pCtx->param[1].i64 = 0; + + return true; +} + +// todo opt for null block +static void first_function(SQLFunctionCtx *pCtx) { + if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) { + return; + } + + int32_t notNullElems = 0; + + // handle the null value + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + memcpy(pCtx->pOutput, data, pCtx->inputBytes); + + TSKEY k = GET_TS_DATA(pCtx, i); + DO_UPDATE_TAG_COLUMNS(pCtx, k); + + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); + pInfo->hasResult = DATA_SET_FLAG; + pInfo->complete = true; + + notNullElems++; + break; + } + + SET_VAL(pCtx, notNullElems, 1); +} + +static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { + if (pCtx->order == TSDB_ORDER_DESC) { + return; + } + + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } SET_VAL(pCtx, 1, 1); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1534,12 +1827,12 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; @@ -1565,7 +1858,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { // find the first not null value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -1583,7 +1876,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { } static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1598,35 +1891,18 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - -static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); - char * pData = GET_INPUT_CHAR(pCtx); + char * pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } // The param[1] is used to keep the initial value of max ts value - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64Key > pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); - pCtx->param[1].i64Key = pInput->ts; + if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 > pInput->ts) { + memcpy(pCtx->pOutput, pData, pCtx->outputBytes); + pCtx->param[1].i64 = pInput->ts; pCtx->param[1].nType = pCtx->outputType; DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); @@ -1645,22 +1921,22 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i64Key || pCtx->preAggVals.dataBlockLoaded == false) { + if (pCtx->order != pCtx->param[0].i64 || pCtx->preAggVals.dataBlockLoaded == false) { return; } int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; } } - memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); + memcpy(pCtx->pOutput, data, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[i]; + TSKEY ts = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1675,21 +1951,21 @@ static void last_function(SQLFunctionCtx *pCtx) { } static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } // the scan order is not the required order, ignore it - if (pCtx->order != pCtx->param[0].i64Key) { + if (pCtx->order != pCtx->param[0].i64) { return; } if (pCtx->order == TSDB_ORDER_DESC) { SET_VAL(pCtx, 1, 1); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -1697,12 +1973,12 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->complete = true; // set query completed } else { // in case of ascending order check, all data needs to be checked SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); *(TSKEY*)buf = ts; DO_UPDATE_TAG_COLUMNS(pCtx, ts); @@ -1711,16 +1987,16 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { #if defined(_DEBUG_VIEW) - tscDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); + qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); #endif - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; @@ -1733,7 +2009,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order != pCtx->param[0].i64Key) { + if (pCtx->order != pCtx->param[0].i64) { return; } @@ -1745,7 +2021,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; @@ -1769,7 +2045,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1778,7 +2054,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order != pCtx->param[0].i64Key) { + if (pCtx->order != pCtx->param[0].i64) { return; } @@ -1787,32 +2063,13 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); } -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - // the input data is null - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - /* * in the secondary merge(local reduce), the output is limited by the * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); +static void last_dist_func_merge(SQLFunctionCtx *pCtx) { + char *pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1823,9 +2080,9 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { * param[1] used to keep the corresponding timestamp to decide if current result is * the true last result */ - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64Key < pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); - pCtx->param[1].i64Key = pInput->ts; + if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 < pInput->ts) { + memcpy(pCtx->pOutput, pData, pCtx->outputBytes); + pCtx->param[1].i64 = pInput->ts; pCtx->param[1].nType = pCtx->outputType; DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); @@ -1841,23 +2098,24 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { */ static void last_row_function(SQLFunctionCtx *pCtx) { assert(pCtx->size >= 1); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); // assign the last element in current data block - assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); + assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the result to final result buffer in case of super table query if (pCtx->stableQuery) { - SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; + SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes); + pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } else { - DO_UPDATE_TAG_COLUMNS(pCtx, pCtx->ptsList[pCtx->size - 1]); + TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); + DO_UPDATE_TAG_COLUMNS(pCtx, ts); } SET_VAL(pCtx, pCtx->size, 1); @@ -1868,9 +2126,9 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->outputType); + setVardataNull(pCtx->pOutput, pCtx->outputType); } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } return; @@ -1884,18 +2142,18 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) { static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, SExtTagsInfo *pTagInfo, int16_t stage) { dst->v.nType = type; - dst->v.i64Key = *(int64_t *)val; + dst->v.i64 = *(int64_t *)val; dst->timestamp = tsKey; int32_t size = 0; - if (stage == SECONDARY_STAGE_MERGE || stage == FIRST_STAGE_MERGE) { + if (stage == MERGE_STAGE) { memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; if (ctx->functionId == TSDB_FUNC_TS_DUMMY) { ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - ctx->tag.i64Key = tsKey; + ctx->tag.i64 = tsKey; } tVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); @@ -1909,28 +2167,31 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 (dst)->timestamp = (src)->timestamp; \ (dst)->v = (src)->v; \ memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ - } while (0); + } while (0) static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); + tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); tValuePair **pList = pInfo->res; assert(pList != NULL); if (pInfo->num < maxLen) { if (pInfo->num == 0 || - ((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && - val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || - ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && - val.dKey >= pList[pInfo->num - 1]->v.dKey)) { - valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); + (IS_SIGNED_NUMERIC_TYPE(type) && val.i64 >= pList[pInfo->num - 1]->v.i64) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 >= pList[pInfo->num - 1]->v.u64) || + (IS_FLOAT_TYPE(type) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) { + valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; - - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { - while (i >= 0 && pList[i]->v.i64Key > val.i64Key) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + while (i >= 0 && pList[i]->v.i64 > val.i64) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i >= 0 && pList[i]->v.u64 > val.u64) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); i -= 1; } @@ -1941,18 +2202,24 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, } } - valuePairAssign(pList[i + 1], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i + 1], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage); } pInfo->num++; } else { int32_t i = 0; - - if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key > pList[0]->v.i64Key) || - ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey > pList[0]->v.dKey)) { + + if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || + (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { // find the appropriate the slot position - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { - while (i + 1 < maxLen && pList[i + 1]->v.i64Key < val.i64Key) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + while (i + 1 < maxLen && pList[i + 1]->v.i64 < val.i64) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; + } + } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i + 1 < maxLen && pList[i + 1]->v.u64 < val.u64) { VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); i += 1; } @@ -1962,8 +2229,8 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, i += 1; } } - - valuePairAssign(pList[i], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); + + valuePairAssign(pList[i], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); } } } @@ -1971,22 +2238,27 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); + tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); tValuePair **pList = pInfo->res; assert(pList != NULL); if (pInfo->num < maxLen) { if (pInfo->num == 0) { - valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { - while (i >= 0 && pList[i]->v.i64Key < val.i64Key) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + while (i >= 0 && pList[i]->v.i64 < val.i64) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); i -= 1; } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i >= 0 && pList[i]->v.u64 < val.u64) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; + } } else { while (i >= 0 && pList[i]->v.dKey < val.dKey) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); @@ -1994,18 +2266,24 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } } - valuePairAssign(pList[i + 1], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i + 1], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); } pInfo->num++; } else { int32_t i = 0; - if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key < pList[0]->v.i64Key) || - ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey < pList[0]->v.dKey)) { + if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || + (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { // find the appropriate the slot position - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { - while (i + 1 < maxLen && pList[i + 1]->v.i64Key > val.i64Key) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + while (i + 1 < maxLen && pList[i + 1]->v.i64 > val.i64) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; + } + } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i + 1 < maxLen && pList[i + 1]->v.u64 > val.u64) { VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); i += 1; } @@ -2016,7 +2294,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } } - valuePairAssign(pList[i], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); } } } @@ -2038,18 +2316,23 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { tValuePair *pLeftElem = *(tValuePair **)pLeft; tValuePair *pRightElem = *(tValuePair **)pRight; - int32_t type = pLeftElem->v.nType; - if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + if (IS_FLOAT_TYPE(pLeftElem->v.nType)) { if (pLeftElem->v.dKey == pRightElem->v.dKey) { return 0; } else { return pLeftElem->v.dKey > pRightElem->v.dKey ? 1 : -1; } + } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){ + if (pLeftElem->v.i64 == pRightElem->v.i64) { + return 0; + } else { + return pLeftElem->v.i64 > pRightElem->v.i64 ? 1 : -1; + } } else { - if (pLeftElem->v.i64Key == pRightElem->v.i64Key) { + if (pLeftElem->v.u64 == pRightElem->v.u64) { return 0; } else { - return pLeftElem->v.i64Key > pRightElem->v.i64Key ? 1 : -1; + return pLeftElem->v.u64 > pRightElem->v.u64 ? 1 : -1; } } } @@ -2066,50 +2349,54 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); switch (type) { + case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: { - int32_t *output = (int32_t *)pCtx->aOutputBuf; + int32_t *output = (int32_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int32_t)tvp[i]->v.i64Key; + *output = (int32_t)tvp[i]->v.i64; } break; } + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: { - int64_t *output = (int64_t *)pCtx->aOutputBuf; + int64_t *output = (int64_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->v.i64Key; + *output = tvp[i]->v.i64; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *output = (double *)pCtx->aOutputBuf; + double *output = (double *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = tvp[i]->v.dKey; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *output = (float *)pCtx->aOutputBuf; + float *output = (float *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = (float)tvp[i]->v.dKey; } break; } + case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: { - int16_t *output = (int16_t *)pCtx->aOutputBuf; + int16_t *output = (int16_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int16_t)tvp[i]->v.i64Key; + *output = (int16_t)tvp[i]->v.i64; } break; } + case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: { - int8_t *output = (int8_t *)pCtx->aOutputBuf; + int8_t *output = (int8_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int8_t)tvp[i]->v.i64Key; + *output = (int8_t)tvp[i]->v.i64; } break; } default: { - tscError("top/bottom function not support data type:%d", pCtx->inputType); + qError("top/bottom function not support data type:%d", pCtx->inputType); return; } } @@ -2124,7 +2411,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { // todo check malloc failure char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - pData[i] = pCtx->tagInfo.pTagCtxList[i]->aOutputBuf; + pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; } for (int32_t i = 0; i < len; ++i, output += step) { @@ -2151,8 +2438,8 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return (STopBotInfo*) pCtx->aOutputBuf; + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { + return (STopBotInfo*) pCtx->pOutput; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer return GET_ROWCELL_INTERBUF(pResInfo); } @@ -2167,7 +2454,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); // required number of results are not reached, continue load data block - if (pTopBotInfo->num < pCtx->param[0].i64Key) { + if (pTopBotInfo->num < pCtx->param[0].i64) { return true; } @@ -2176,13 +2463,13 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha if (functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(maxval) > pRes[0]->v.i64Key; + return GET_INT8_VAL(maxval) > pRes[0]->v.i64; case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(maxval) > pRes[0]->v.i64Key; + return GET_INT16_VAL(maxval) > pRes[0]->v.i64; case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(maxval) > pRes[0]->v.i64Key; + return GET_INT32_VAL(maxval) > pRes[0]->v.i64; case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(maxval) > pRes[0]->v.i64Key; + return GET_INT64_VAL(maxval) > pRes[0]->v.i64; case TSDB_DATA_TYPE_FLOAT: return GET_FLOAT_VAL(maxval) > pRes[0]->v.dKey; case TSDB_DATA_TYPE_DOUBLE: @@ -2193,13 +2480,13 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha } else { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(minval) < pRes[0]->v.i64Key; + return GET_INT8_VAL(minval) < pRes[0]->v.i64; case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(minval) < pRes[0]->v.i64Key; + return GET_INT16_VAL(minval) < pRes[0]->v.i64; case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(minval) < pRes[0]->v.i64Key; + return GET_INT32_VAL(minval) < pRes[0]->v.i64; case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(minval) < pRes[0]->v.i64Key; + return GET_INT64_VAL(minval) < pRes[0]->v.i64; case TSDB_DATA_TYPE_FLOAT: return GET_FLOAT_VAL(minval) < pRes[0]->v.dKey; case TSDB_DATA_TYPE_DOUBLE: @@ -2219,11 +2506,12 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; - tmp += POINTER_BYTES * pCtx->param[0].i64Key; + tmp += POINTER_BYTES * pCtx->param[0].i64; size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - - for (int32_t i = 0; i < pCtx->param[0].i64Key; ++i) { +// assert(pCtx->param[0].i64 > 0); + + for (int32_t i = 0; i < pCtx->param[0].i64; ++i) { pTopBotInfo->res[i] = (tValuePair*) tmp; pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); tmp += size; @@ -2248,13 +2536,15 @@ static void top_function(SQLFunctionCtx *pCtx) { assert(pRes->num >= 0); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } notNullElems++; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2271,7 +2561,7 @@ static void top_function(SQLFunctionCtx *pCtx) { } static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2280,36 +2570,16 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { assert(pRes->num >= 0); SET_VAL(pCtx, 1, 1); - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + TSKEY ts = GET_TS_DATA(pCtx, index); + + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void top_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void top_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2319,7 +2589,7 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->outputType; - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, + do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp, type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2336,15 +2606,20 @@ static void bottom_function(SQLFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); + if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { + buildTopBotStruct(pRes, pCtx); + } + for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; + continue; } notNullElems++; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2361,43 +2636,28 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY ts = GET_TS_DATA(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } STopBotInfo *pRes = getTopBotOutputInfo(pCtx); + + if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { + buildTopBotStruct(pRes, pCtx); + } + SET_VAL(pCtx, 1, 1); - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, - NULL, 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void bottom_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2407,7 +2667,7 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->outputType; - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, type, + do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp, type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2433,11 +2693,11 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { tValuePair **tvp = pRes->res; // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; + if (pCtx->param[1].i64 == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; + } else if (pCtx->param[1].i64 > PRIMARYKEY_TIMESTAMP_COL_INDEX) { + __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } @@ -2473,15 +2733,19 @@ static void percentile_function(SQLFunctionCtx *pCtx) { if (pInfo->stage == 0) { if (pCtx->preAggVals.isSet) { double tmin = 0.0, tmax = 0.0; - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); - tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_FLOAT_TYPE(pCtx->inputType)) { + tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); + tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max); } else { assert(true); } + if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { SET_DOUBLE_VAL(&pInfo->minval, tmin); } @@ -2493,7 +2757,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); } else { for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2518,7 +2782,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { // the second stage, calculate the true percentile value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2532,7 +2796,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { } static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2564,7 +2828,7 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void percentile_finalizer(SQLFunctionCtx *pCtx) { - double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; + double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); @@ -2572,9 +2836,9 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { tMemBucket * pMemBucket = ppInfo->pMemBucket; if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null assert(ppInfo->numOfElems == 0); - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } else { - *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); + *(double *)pCtx->pOutput = getPercentile(pMemBucket, v); } tMemBucketDestroy(pMemBucket); @@ -2589,24 +2853,34 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { // all data are null, set it completed if (pInfo->numOfElems == 0) { pResInfo->complete = true; + } else { + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); } pInfo->stage += 1; - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval)); } else { pResInfo->complete = true; } } ////////////////////////////////////////////////////////////////////////////////// +static void buildHistogramInfo(SAPercentileInfo* pInfo) { + pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); + pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); +} + static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return (SAPercentileInfo*) pCtx->aOutputBuf; + SAPercentileInfo* pInfo = NULL; + + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { + pInfo = (SAPercentileInfo*) pCtx->pOutput; } else { - return GET_ROWCELL_INTERBUF(pResInfo); + pInfo = GET_ROWCELL_INTERBUF(pResInfo); } + + buildHistogramInfo(pInfo); + return pInfo; } static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { @@ -2626,9 +2900,11 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); + + assert(pInfo->pHisto->elems != NULL); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2652,7 +2928,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2670,42 +2946,7 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - size_t size = sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); //(SAPercentileInfo *)pCtx->aOutputBuf; - SHistogramInfo * pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, size); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - tHistogramDestroy(&pRes); - } - - SET_VAL(pCtx, 1, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); @@ -2735,22 +2976,22 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { } static void apercentile_finalizer(SQLFunctionCtx *pCtx) { - double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey; + double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64 : pCtx->param[0].dKey; SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null assert(pOutput->pHisto->numOfElems > 0); double ratio[] = {v}; double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->aOutputBuf, res, sizeof(double)); + memcpy(pCtx->pOutput, res, sizeof(double)); free(res); } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } } else { @@ -2758,10 +2999,10 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double ratio[] = {v}; double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->aOutputBuf, res, sizeof(double)); + memcpy(pCtx->pOutput, res, sizeof(double)); free(res); } else { // no need to free - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } } @@ -2808,7 +3049,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { double(*param)[3] = pInfo->mat; double x = pInfo->startVal; - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); int32_t numOfElem = 0; switch (pCtx->inputType) { @@ -2829,17 +3070,17 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { numOfElem++; } break; - }; + } case TSDB_DATA_TYPE_BIGINT: { int64_t *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_DOUBLE: { double *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_FLOAT: { float *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); @@ -2849,12 +3090,32 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { int16_t *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_TINYINT: { int8_t *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } } pInfo->startVal = x; @@ -2868,7 +3129,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { } static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2883,7 +3144,7 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { int32_t *p = pData; LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_TINYINT: { int8_t *p = pData; LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); @@ -2910,7 +3171,7 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { break; } default: - tscError("error data type in leastsquare function:%d", pCtx->inputType); + qError("error data type in leastsquare function:%d", pCtx->inputType); }; SET_VAL(pCtx, 1, 1); @@ -2927,7 +3188,7 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } @@ -2946,16 +3207,16 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { param[1][2] /= param[1][1]; int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; - size_t n = snprintf(varDataVal(pCtx->aOutputBuf), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", + size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", param[0][2], param[1][2]); - varDataSetLen(pCtx->aOutputBuf, n); + varDataSetLen(pCtx->pOutput, n); doFinalizer(pCtx); } static void date_col_output_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp; + *(int64_t *)(pCtx->pOutput) = pCtx->startTs; } static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -2970,17 +3231,17 @@ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, + memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, pCtx->inputBytes); } } - pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; + pCtx->pOutput += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -2990,15 +3251,15 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { } // only one output - if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { + if (pCtx->param[0].i64 == 1 && pResInfo->numOfRes >= 1) { return; } INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + char *pData = GET_INPUT_DATA(pCtx, index); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pCtx->aOutputBuf += pCtx->inputBytes; + pCtx->pOutput += pCtx->inputBytes; } /** @@ -3011,22 +3272,22 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { assert(pCtx->inputBytes == pCtx->outputBytes); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); - char* data = pCtx->aOutputBuf; - pCtx->aOutputBuf += pCtx->outputBytes; + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); + char* data = pCtx->pOutput; + pCtx->pOutput += pCtx->outputBytes; // directly copy from the first one for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->aOutputBuf, data, pCtx->outputBytes); - pCtx->aOutputBuf += pCtx->outputBytes; + memmove(pCtx->pOutput, data, pCtx->outputBytes); + pCtx->pOutput += pCtx->outputBytes; } } static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true); - pCtx->aOutputBuf += pCtx->outputBytes; + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->tag.nType, true); + pCtx->pOutput += pCtx->outputBytes; } /** @@ -3038,19 +3299,19 @@ static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { */ static void tag_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); } static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); } static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - char *pData = GET_INPUT_CHAR(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + char *pData = GET_INPUT_DATA_LIST(pCtx); + assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); } enum { @@ -3069,7 +3330,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { // TODO difference in date column static void diff_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_CHAR(pCtx); + void *data = GET_INPUT_DATA_LIST(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); int32_t notNullElems = 0; @@ -3077,12 +3338,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - TSKEY * pTimestamp = pCtx->ptsOutputBuf; - + TSKEY* pTimestamp = pCtx->ptsOutputBuf; + TSKEY* tsList = GET_TS_LIST(pCtx); + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *pData = (int32_t *)data; - int32_t *pOutput = (int32_t *)pCtx->aOutputBuf; + int32_t *pOutput = (int32_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3090,23 +3352,23 @@ static void diff_function(SQLFunctionCtx *pCtx) { } if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { - *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64); + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { - *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); // direct previous may be null - *pTimestamp = pCtx->ptsList[i]; + *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; } @@ -3114,7 +3376,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { }; case TSDB_DATA_TYPE_BIGINT: { int64_t *pData = (int64_t *)data; - int64_t *pOutput = (int64_t *)pCtx->aOutputBuf; + int64_t *pOutput = (int64_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3122,31 +3384,31 @@ static void diff_function(SQLFunctionCtx *pCtx) { } if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { - *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pOutput = pData[i] - pCtx->param[1].i64; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { - *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pOutput = pData[i] - pCtx->param[1].i64; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; - }; + } case TSDB_DATA_TYPE_DOUBLE: { double *pData = (double *)data; - double *pOutput = (double *)pCtx->aOutputBuf; + double *pOutput = (double *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3158,12 +3420,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } @@ -3173,10 +3435,10 @@ static void diff_function(SQLFunctionCtx *pCtx) { notNullElems++; } break; - }; + } case TSDB_DATA_TYPE_FLOAT: { float *pData = (float *)data; - float *pOutput = (float *)pCtx->aOutputBuf; + float *pOutput = (float *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3188,13 +3450,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3206,10 +3468,10 @@ static void diff_function(SQLFunctionCtx *pCtx) { notNullElems++; } break; - }; + } case TSDB_DATA_TYPE_SMALLINT: { int16_t *pData = (int16_t *)data; - int16_t *pOutput = (int16_t *)pCtx->aOutputBuf; + int16_t *pOutput = (int16_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3217,30 +3479,30 @@ static void diff_function(SQLFunctionCtx *pCtx) { } if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { - *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64); + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { - *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64); + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; - }; + } case TSDB_DATA_TYPE_TINYINT: { int8_t *pData = (int8_t *)data; - int8_t *pOutput = (int8_t *)pCtx->aOutputBuf; + int8_t *pOutput = (int8_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { @@ -3248,30 +3510,30 @@ static void diff_function(SQLFunctionCtx *pCtx) { } if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { - *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64); + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { - *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64); + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].i64 = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; - }; + } default: - tscError("error input type"); + qError("error input type"); } // initial value is not set yet @@ -3286,7 +3548,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += forwardStep; - pCtx->aOutputBuf += forwardStep * pCtx->outputBytes; + pCtx->pOutput += forwardStep * pCtx->outputBytes; pCtx->ptsOutputBuf = (char*)pCtx->ptsOutputBuf + forwardStep * TSDB_KEYSIZE; } } @@ -3295,16 +3557,16 @@ static void diff_function(SQLFunctionCtx *pCtx) { do { \ if ((ctx)->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { \ (ctx)->param[1].nType = (ctx)->inputType; \ - *(type *)&(ctx)->param[1].i64Key = *(type *)(d); \ + *(type *)&(ctx)->param[1].i64 = *(type *)(d); \ } else { \ - *(type *)(ctx)->aOutputBuf = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64Key)); \ - *(type *)(&(ctx)->param[1].i64Key) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = (ctx)->ptsList[index]; \ + *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64)); \ + *(type *)(&(ctx)->param[1].i64) = *(type *)(d); \ + *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ } \ } while (0); static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3320,11 +3582,11 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { case TSDB_DATA_TYPE_INT: { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].nType = pCtx->inputType; - pCtx->param[1].i64Key = *(int32_t *)pData; + pCtx->param[1].i64 = *(int32_t *)pData; } else { - *(int32_t *)pCtx->aOutputBuf = *(int32_t *)pData - (int32_t)pCtx->param[1].i64Key; - pCtx->param[1].i64Key = *(int32_t *)pData; - *(int64_t *)pCtx->ptsOutputBuf = pCtx->ptsList[index]; + *(int32_t *)pCtx->pOutput = *(int32_t *)pData - (int32_t)pCtx->param[1].i64; + pCtx->param[1].i64 = *(int32_t *)pData; + *(int64_t *)pCtx->ptsOutputBuf = GET_TS_DATA(pCtx, index); } break; }; @@ -3349,11 +3611,11 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { break; }; default: - tscError("error input type"); + qError("error input type"); } if (GET_RES_INFO(pCtx)->numOfRes > 0) { - pCtx->aOutputBuf += pCtx->outputBytes * step; + pCtx->pOutput += pCtx->outputBytes * step; pCtx->ptsOutputBuf = (char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * step; } } @@ -3377,10 +3639,8 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); - - pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; - pCtx->param[1].pz = NULL; + arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); + pCtx->pOutput += pCtx->outputBytes * pCtx->size; } static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -3388,9 +3648,9 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, 1, pCtx->pOutput, sas, pCtx->order, getArithColumnData); - pCtx->aOutputBuf += pCtx->outputBytes; + pCtx->pOutput += pCtx->outputBytes; } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ @@ -3419,7 +3679,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { pCtx->param[0].dKey = DBL_MAX; pCtx->param[3].dKey = -DBL_MAX; } else { @@ -3445,17 +3705,17 @@ static void spread_function(SQLFunctionCtx *pCtx) { if (numOfElems == 0) { goto _spread_over; } - - if ((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) || + + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { if (pInfo->min > pCtx->preAggVals.statis.min) { pInfo->min = (double)pCtx->preAggVals.statis.min; } - + if (pInfo->max < pCtx->preAggVals.statis.max) { pInfo->max = (double)pCtx->preAggVals.statis.max; } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + } else if (IS_FLOAT_TYPE(pCtx->inputType)) { if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.min))) { pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.min)); } @@ -3468,7 +3728,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { goto _spread_over; } - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); numOfElems = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -3483,6 +3743,14 @@ static void spread_function(SQLFunctionCtx *pCtx) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems); } if (!pCtx->hasNull) { @@ -3499,12 +3767,12 @@ static void spread_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3534,48 +3802,15 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { pInfo->max = val; } - if (val < pInfo->min) { - pInfo->min = val; - } - - pResInfo->hasResult = DATA_SET_FLAG; - pInfo->hasResult = DATA_SET_FLAG; - - if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - } -} - -void spread_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t notNullElems = 0; - for (int32_t i = 0; i < pCtx->size; ++i) { - SSpreadInfo *input = (SSpreadInfo *)GET_INPUT_CHAR_INDEX(pCtx, i); - - /* no assign tag, the value is null */ - if (input->hasResult != DATA_SET_FLAG) { - continue; - } - - if (pResData->min > input->min) { - pResData->min = input->min; - } - - if (pResData->max < input->max) { - pResData->max = input->max; - } - - pResData->hasResult = DATA_SET_FLAG; - notNullElems++; - } - - if (notNullElems > 0) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - pResInfo->hasResult = DATA_SET_FLAG; + if (val < pInfo->min) { + pInfo->min = val; + } + + pResInfo->hasResult = DATA_SET_FLAG; + pInfo->hasResult = DATA_SET_FLAG; + + if (pCtx->stableQuery) { + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } @@ -3583,8 +3818,8 @@ void spread_func_merge(SQLFunctionCtx *pCtx) { * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer */ -void spread_func_sec_merge(SQLFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_CHAR(pCtx); +void spread_func_merge(SQLFunctionCtx *pCtx) { + SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); if (pData->hasResult != DATA_SET_FLAG) { return; } @@ -3607,26 +3842,25 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { */ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } - *(double *)pCtx->aOutputBuf = pCtx->param[3].dKey - pCtx->param[0].dKey; + *(double *)pCtx->pOutput = pCtx->param[3].dKey - pCtx->param[0].dKey; } else { - assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); + assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } - *(double *)pCtx->aOutputBuf = pInfo->max - pInfo->min; + *(double *)pCtx->pOutput = pInfo->max - pInfo->min; } GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case @@ -3647,43 +3881,53 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->lastKey = INT64_MIN; + pInfo->p.key = INT64_MIN; pInfo->win = TSWINDOW_INITIALIZER; return true; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { +static double twa_get_area(SPoint1 s, SPoint1 e) { + if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { + return (s.val + e.val) * (e.key - s.key) / 2; + } + + double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); + double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; + return val; +} + +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *tsList = GET_TS_LIST(pCtx); int32_t i = index; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); + SPoint1* last = &pInfo->p; if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); + assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - assert(pInfo->lastKey == INT64_MIN); + assert(last->key == INT64_MIN); - pInfo->lastKey = primaryKey[tsIndex + i]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); + pInfo->dOutput += twa_get_area(pCtx->start, *last); pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = pCtx->start.key; notNullElems++; i += step; - } else if (pInfo->lastKey == INT64_MIN) { - pInfo->lastKey = primaryKey[tsIndex + i]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + } else if (pInfo->p.key == INT64_MIN) { + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->lastKey; + pInfo->win.skey = last->key; notNullElems++; i += step; } @@ -3691,236 +3935,132 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t // calculate the value of switch(pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + tsIndex]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + tsIndex]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + tsIndex]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); - pInfo->lastValue = (double) val[i]; - pInfo->lastKey = primaryKey[i + tsIndex]; + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0); + float *val = (float*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + tsIndex]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0); + double *val = (double*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + tsIndex]; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); - pInfo->lastValue = pCtx->end.val; - pInfo->lastKey = pCtx->end.key; - } - - pInfo->win.ekey = pInfo->lastKey; - return notNullElems; -} - -static void twa_function(SQLFunctionCtx *pCtx) { - void * data = GET_INPUT_CHAR(pCtx); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // skip null value - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i += step; - } - - int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size); - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); - } -} - -static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - int32_t i = pCtx->startOffset; - int32_t size = pCtx->size; - - if (pCtx->start.key != INT64_MIN) { - assert(pInfo->lastKey == INT64_MIN); - - pInfo->lastKey = primaryKey[index]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += 1; - } else if (pInfo->lastKey == INT64_MIN) { - pInfo->lastKey = primaryKey[index]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->lastKey; - notNullElems++; - i += 1; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + index]; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + index]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + index]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); - pInfo->lastValue = (double) val[i]; - pInfo->lastKey = primaryKey[i + index]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + case TSDB_DATA_TYPE_UINT: { + uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + index]; + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); - pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i + index]; + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; } break; } @@ -3929,49 +4069,60 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the last interpolated time window value if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); - pInfo->lastValue = pCtx->end.val; - pInfo->lastKey = pCtx->end.key; + pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); + pInfo->p = pCtx->end; + } + + pInfo->win.ekey = pInfo->p.key; + return notNullElems; +} + +static void twa_function(SQLFunctionCtx *pCtx) { + void *data = GET_INPUT_DATA_LIST(pCtx); + + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + // skip null value + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); + int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); + while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { + i += step; } - pInfo->win.ekey = pInfo->lastKey; + int32_t notNullElems = 0; + if (i >= 0 && i < pCtx->size) { + notNullElems = twa_function_impl(pCtx, i, pCtx->size); + } SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } - + if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); + memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo)); } } -static void twa_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; - char * indicator = pCtx->aInputElemBuf; - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { - STwaInfo *pInput = (STwaInfo*) indicator; - - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } - - numOfNotNull++; - pBuf->dOutput += pInput->dOutput; +static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { + void *pData = GET_INPUT_DATA(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; + } + + int32_t notNullElems = twa_function_impl(pCtx, index, 1); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + + SET_VAL(pCtx, notNullElems, 1); - pBuf->win = pInput->win; - pBuf->lastKey = pInput->lastKey; + if (notNullElems > 0) { + pResInfo->hasResult = DATA_SET_FLAG; } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; + + if (pCtx->stableQuery) { + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } @@ -3984,25 +4135,24 @@ void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); + pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - assert(pInfo->win.ekey == pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - + + assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); if (pInfo->win.ekey == pInfo->win.skey) { - *(double *)pCtx->aOutputBuf = pInfo->lastValue; + *(double *)pCtx->pOutput = pInfo->p.val; } else { - *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); + *(double *)pCtx->pOutput = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); } GET_RES_INFO(pCtx)->numOfRes = 1; @@ -4013,87 +4163,73 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { * * @param pCtx */ -static void interp_function(SQLFunctionCtx *pCtx) { - // at this point, the value is existed, return directly - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - if (pCtx->size == 1) { - char *pData = GET_INPUT_CHAR(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); +static void interp_function_impl(SQLFunctionCtx *pCtx) { + int32_t type = (int32_t) pCtx->param[2].i64; + if (type == TSDB_FILL_NONE) { + return; + } + + if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { + *(TSKEY *) pCtx->pOutput = pCtx->startTs; } else { - /* - * use interpolation to generate the result. - * Note: the result of primary timestamp column uses the timestamp specified by user in the query sql - */ - assert(pCtx->size == 2); - if (pInfo->type == TSDB_FILL_NONE) { // set no output result + if (pCtx->start.key == INT64_MIN) { + assert(pCtx->end.key == INT64_MIN); return; } - - if (pInfo->primaryCol == 1) { - *(TSKEY *) pCtx->aOutputBuf = pInfo->ts; - } else { - if (pInfo->type == TSDB_FILL_NULL) { - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->outputType); - } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - } - - SET_VAL(pCtx, pCtx->size, 1); - } else if (pInfo->type == TSDB_FILL_SET_VALUE) { - tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType, true); - } else if (pInfo->type == TSDB_FILL_PREV) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, 0); - assignVal(pCtx->aOutputBuf, data, pCtx->outputBytes, pCtx->outputType); - - SET_VAL(pCtx, pCtx->size, 1); - } else if (pInfo->type == TSDB_FILL_LINEAR) { - char *data1 = GET_INPUT_CHAR_INDEX(pCtx, 0); - char *data2 = GET_INPUT_CHAR_INDEX(pCtx, 1); - - TSKEY key1 = pCtx->ptsList[0]; - TSKEY key2 = pCtx->ptsList[1]; - - SPoint point1 = {.key = key1, .val = data1}; - SPoint point2 = {.key = key2, .val = data2}; - - SPoint point = {.key = pInfo->ts, .val = pCtx->aOutputBuf}; - - int32_t srcType = pCtx->inputType; - if ((srcType >= TSDB_DATA_TYPE_TINYINT && srcType <= TSDB_DATA_TYPE_BIGINT) || - srcType == TSDB_DATA_TYPE_TIMESTAMP || srcType == TSDB_DATA_TYPE_DOUBLE) { - point1.val = data1; - point2.val = data2; - - if (isNull(data1, srcType) || isNull(data2, srcType)) { - setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); - } else { - taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); - } - } else if (srcType == TSDB_DATA_TYPE_FLOAT) { - point1.val = data1; - point2.val = data2; - - if (isNull(data1, srcType) || isNull(data2, srcType)) { - setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); - } else { - taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); - } - + + if (type == TSDB_FILL_NULL) { + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + } else if (type == TSDB_FILL_SET_VALUE) { + tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); + } else if (type == TSDB_FILL_PREV) { + if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { + SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); + } else { + assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType); + } + } else if (type == TSDB_FILL_NEXT) { + if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { + SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); + } else { + assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType); + } + } else if (type == TSDB_FILL_LINEAR) { + SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; + SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; + SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; + + int32_t srcType = pCtx->inputType; + if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? + if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } else { - if (srcType == TSDB_DATA_TYPE_BINARY || srcType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->inputType); - } else { - setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); - } + taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); } + } else { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } } } - - SET_VAL(pCtx, pCtx->size, 1); + + SET_VAL(pCtx, 1, 1); + +} +static void interp_function(SQLFunctionCtx *pCtx) { + // at this point, the value is existed, return directly + if (pCtx->size > 0) { + // impose the timestamp check + TSKEY key = GET_TS_DATA(pCtx, 0); + if (key == pCtx->startTs) { + char *pData = GET_INPUT_DATA(pCtx, 0); + assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); + SET_VAL(pCtx, 1, 1); + } else { + interp_function_impl(pCtx); + } + } else { //no qualified data rows and interpolation is required + interp_function_impl(pCtx); + } } static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { @@ -4113,15 +4249,15 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - const char *input = GET_INPUT_CHAR(pCtx); + const char *input = GET_INPUT_DATA_LIST(pCtx); // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); + char *d = GET_INPUT_DATA(pCtx, i); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -4130,7 +4266,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { } static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4140,7 +4276,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; @@ -4153,8 +4289,10 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) { STSBuf * pTSbuf = pInfo->pTSBuf; tsBufFlush(pTSbuf); - strcpy(pCtx->aOutputBuf, pTSbuf->path); + *(FILE **)pCtx->pOutput = pTSbuf->f; + + pTSbuf->remainOpen = true; tsBufDestroy(pTSbuf); doFinalizer(pCtx); } @@ -4186,19 +4324,18 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { double resultVal = ((double)diff) / duration; - tscDebug("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " resultVal:%f", + qDebug("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " resultVal:%f", pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, resultVal); return resultVal; } - static bool rate_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->pOutput + pCtx->outputBytes; SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->CorrectionValue = 0; @@ -4214,20 +4351,19 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return true; } - static void rate_function(SQLFunctionCtx *pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + int32_t notNullElems = 0; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); - tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); + qDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - tscDebug("%p rate_function() index of null data:%d", pCtx, i); + qDebug("%p rate_function() index of null data:%d", pCtx, i); continue; } @@ -4240,19 +4376,19 @@ static void rate_function(SQLFunctionCtx *pCtx) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - tscDebug("firstValue:%" PRId64 " firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); + qDebug("firstValue:%" PRId64 " firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); } if (INT64_MIN == pRateInfo->lastValue) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { pRateInfo->CorrectionValue += pRateInfo->lastValue; - tscDebug("CorrectionValue:%" PRId64, pRateInfo->CorrectionValue); + qDebug("CorrectionValue:%" PRId64, pRateInfo->CorrectionValue); } pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; - tscDebug("lastValue:%" PRId64 " lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); + qDebug("lastValue:%" PRId64 " lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); } if (!pCtx->hasNull) { @@ -4268,12 +4404,12 @@ static void rate_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4281,8 +4417,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // NOTE: keep the intermediate result into the interResultBuf SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - + TSKEY *primaryKey = GET_TS_LIST(pCtx); + int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4300,7 +4436,7 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - tscDebug("====%p rate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " CorrectionValue:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->CorrectionValue); + qDebug("====%p rate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " CorrectionValue:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->CorrectionValue); SET_VAL(pCtx, 1, 1); @@ -4310,75 +4446,37 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - - - -static void rate_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - tscDebug("rate_func_merge() size:%d", pCtx->size); - - SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; - char *indicator = pCtx->aInputElemBuf; - - assert(1 == pCtx->size); - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(SRateInfo)) { - SRateInfo *pInput = (SRateInfo *)indicator; - if (DATA_SET_FLAG != pInput->hasResult) { - continue; - } - - numOfNotNull++; - memcpy(pBuf, pInput, sizeof(SRateInfo)); - tscDebug("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, - pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); - } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } - - return; } - - static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); + pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; - SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; - tscDebug("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + SRateInfo* pRateInfo = (SRateInfo*)pCtx->pInput; + qDebug("%p rate_func_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } - - static void rate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - tscDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + qDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); + *(double*)pCtx->pOutput = do_calc_rate(pRateInfo); - tscDebug("rate_finalizer() output result:%f", *(double *)pCtx->aOutputBuf); + qDebug("rate_finalizer() output result:%f", *(double *)pCtx->pOutput); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; @@ -4387,24 +4485,23 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } - static void irate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - - tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); + TSKEY *primaryKey = GET_TS_LIST(pCtx); + + qDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); if (pCtx->size < 1) { return; } for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - tscDebug("%p irate_function() index of null data:%d", pCtx, i); + qDebug("%p irate_function() index of null data:%d", pCtx, i); continue; } @@ -4418,7 +4515,7 @@ static void irate_function(SQLFunctionCtx *pCtx) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; - tscDebug("%p irate_function() lastValue:%" PRId64 " lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); + qDebug("%p irate_function() lastValue:%" PRId64 " lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); continue; } @@ -4426,7 +4523,7 @@ static void irate_function(SQLFunctionCtx *pCtx) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - tscDebug("%p irate_function() firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); + qDebug("%p irate_function() firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); break; } } @@ -4440,20 +4537,20 @@ static void irate_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4464,7 +4561,7 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - tscDebug("====%p irate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->firstValue , pRateInfo->firstKey); + qDebug("====%p irate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->firstValue , pRateInfo->firstKey); SET_VAL(pCtx, 1, 1); @@ -4474,7 +4571,7 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4483,12 +4580,12 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); + char * input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SRateInfo *pInput = (SRateInfo *)input; - tscDebug("%p do_sumrate_merge() hasResult:%d input num:%" PRId64 " input sum:%f total num:%" PRId64 " total sum:%f", pCtx, pInput->hasResult, pInput->num, pInput->sum, pRateInfo->num, pRateInfo->sum); + qDebug("%p do_sumrate_merge() hasResult:%d input num:%" PRId64 " input sum:%f total num:%" PRId64 " total sum:%f", pCtx, pInput->hasResult, pInput->num, pInput->sum, pRateInfo->num, pRateInfo->sum); if (pInput->hasResult != DATA_SET_FLAG) { continue; @@ -4506,17 +4603,12 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { if (DATA_SET_FLAG == pRateInfo->hasResult) { pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, pRateInfo->num, 1); - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void sumrate_func_merge(SQLFunctionCtx *pCtx) { - tscDebug("%p sumrate_func_merge() process ...", pCtx); - do_sumrate_merge(pCtx); -} - -static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { - tscDebug("%p sumrate_func_second_merge() process ...", pCtx); + qDebug("%p sumrate_func_merge() process ...", pCtx); do_sumrate_merge(pCtx); } @@ -4524,20 +4616,20 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); + qDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } if (pRateInfo->num == 0) { // from meter - *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); + *(double*)pCtx->pOutput = do_calc_rate(pRateInfo); } else if (pCtx->functionId == TSDB_FUNC_SUM_RATE || pCtx->functionId == TSDB_FUNC_SUM_IRATE) { - *(double*)pCtx->aOutputBuf = pRateInfo->sum; + *(double*)pCtx->pOutput = pRateInfo->sum; } else { - *(double*)pCtx->aOutputBuf = pRateInfo->sum / pRateInfo->num; + *(double*)pCtx->pOutput = pRateInfo->sum / pRateInfo->num; } pResInfo->numOfRes = 1; @@ -4571,7 +4663,7 @@ int32_t functionCompatList[] = { 1, 1, 1, 1, }; -SQLAggFuncElem aAggs[] = {{ +SAggFunctionInfo aAggs[] = {{ // 0, count function does not invoke the finalize function "count", TSDB_FUNC_COUNT, @@ -4583,7 +4675,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, count_func_merge, - count_func_merge, count_load_data_info, }, { @@ -4598,7 +4689,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, sum_func_merge, - sum_func_second_merge, statisRequired, }, { @@ -4613,7 +4703,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, avg_finalizer, avg_func_merge, - avg_func_second_merge, statisRequired, }, { @@ -4628,7 +4717,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, min_func_merge, - min_func_second_merge, statisRequired, }, { @@ -4643,14 +4731,13 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, max_func_merge, - max_func_second_merge, statisRequired, }, { // 5 "stddev", TSDB_FUNC_STDDEV, - TSDB_FUNC_INVALID_ID, + TSDB_FUNC_STDDEV_DST, TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, function_setup, stddev_function, @@ -4658,7 +4745,6 @@ SQLAggFuncElem aAggs[] = {{ stddev_next_step, stddev_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4673,7 +4759,6 @@ SQLAggFuncElem aAggs[] = {{ percentile_next_step, percentile_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4688,7 +4773,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, apercentile_finalizer, apercentile_func_merge, - apercentile_func_second_merge, dataBlockRequired, }, { @@ -4703,7 +4787,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, firstFuncRequired, }, { @@ -4718,7 +4801,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, lastFuncRequired, }, { @@ -4733,8 +4815,7 @@ SQLAggFuncElem aAggs[] = {{ noop2, no_next_step, last_row_finalizer, - noop1, - last_dist_func_second_merge, + last_dist_func_merge, dataBlockRequired, }, { @@ -4750,7 +4831,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, top_func_merge, - top_func_second_merge, dataBlockRequired, }, { @@ -4766,7 +4846,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, bottom_func_merge, - bottom_func_second_merge, dataBlockRequired, }, { @@ -4781,7 +4860,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, spread_function_finalizer, spread_func_merge, - spread_func_sec_merge, count_load_data_info, }, { @@ -4795,7 +4873,6 @@ SQLAggFuncElem aAggs[] = {{ twa_function_f, no_next_step, twa_function_finalizer, - twa_func_merge, twa_function_copy, dataBlockRequired, }, @@ -4811,7 +4888,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, leastsquares_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4826,7 +4902,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4841,7 +4916,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4856,7 +4930,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4871,7 +4944,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, ts_comp_finalize, copy_function, - copy_function, dataBlockRequired, }, { @@ -4886,7 +4958,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4901,7 +4972,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4916,7 +4986,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4931,7 +5000,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4946,7 +5014,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, noop1, - noop1, dataBlockRequired, }, // distributed version used in two-stage aggregation processes @@ -4962,7 +5029,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, first_dist_func_merge, - first_dist_func_second_merge, firstDistFuncRequired, }, { @@ -4977,11 +5043,24 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, last_dist_func_merge, - last_dist_func_second_merge, lastDistFuncRequired, }, { // 27 + "stddev", // return table id and the corresponding tags for join match and subscribe + TSDB_FUNC_STDDEV_DST, + TSDB_FUNC_AVG, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, + function_setup, + stddev_dst_function, + stddev_dst_function_f, + no_next_step, + stddev_dst_finalizer, + stddev_dst_merge, + dataBlockRequired, + }, + { + // 28 "interp", TSDB_FUNC_INTERP, TSDB_FUNC_INTERP, @@ -4991,12 +5070,11 @@ SQLAggFuncElem aAggs[] = {{ do_sum_f, // todo filter handle no_next_step, doFinalizer, - noop1, copy_function, dataBlockRequired, }, { - // 28 + // 29 "rate", TSDB_FUNC_RATE, TSDB_FUNC_RATE, @@ -5006,12 +5084,11 @@ SQLAggFuncElem aAggs[] = {{ rate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, { - // 29 + // 30 "irate", TSDB_FUNC_IRATE, TSDB_FUNC_IRATE, @@ -5021,12 +5098,11 @@ SQLAggFuncElem aAggs[] = {{ irate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, { - // 30 + // 31 "sum_rate", TSDB_FUNC_SUM_RATE, TSDB_FUNC_SUM_RATE, @@ -5037,11 +5113,10 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { - // 31 + // 32 "sum_irate", TSDB_FUNC_SUM_IRATE, TSDB_FUNC_SUM_IRATE, @@ -5052,11 +5127,10 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { - // 32 + // 33 "avg_rate", TSDB_FUNC_AVG_RATE, TSDB_FUNC_AVG_RATE, @@ -5067,11 +5141,10 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { - // 33 + // 34 "avg_irate", TSDB_FUNC_AVG_IRATE, TSDB_FUNC_AVG_IRATE, @@ -5082,11 +5155,10 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { - // 34 + // 35 "tid_tag", // return table id and the corresponding tags for join match and subscribe TSDB_FUNC_TID_TAG, TSDB_FUNC_TID_TAG, @@ -5097,6 +5169,5 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, noop1, noop1, - noop1, dataBlockRequired, - }}; + } }; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1cde31cfd2646627806fb3fd1cd940ac90dbc71d..f8119b0d4ad07ea235f8e098149a8703a416427e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -15,16 +15,14 @@ #include "os.h" #include "qFill.h" #include "taosmsg.h" -#include "tcache.h" #include "tglobal.h" #include "exception.h" #include "hash.h" -#include "qAst.h" +#include "texpr.h" #include "qExecutor.h" #include "qResultbuf.h" #include "qUtil.h" -#include "query.h" #include "queryLog.h" #include "tlosertree.h" #include "ttype.h" @@ -35,8 +33,6 @@ * check if the primary column is load by default, otherwise, the program will * forced to load primary column explicitly. */ -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) -#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -53,28 +49,7 @@ #define TIME_WINDOW_COPY(_dst, _src) do {\ (_dst).skey = (_src).skey;\ (_dst).ekey = (_src).ekey;\ -} while (0); - -enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, - - /* result output buffer is full, current query is paused. - * this status is only exist in group-by clause and diff/add/division/multiply/ query. - */ - QUERY_RESBUF_FULL = 0x2u, - - /* query is over - * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. - * 2. when all data within queried time window, it is also denoted as query_completed - */ - QUERY_COMPLETED = 0x4u, - - /* when the result is not completed return to client, this status will be - * usually used in case of interval query with interpolation option - */ - QUERY_OVER = 0x8u, -}; +} while (0) enum { TS_JOIN_TS_EQUAL = 0, @@ -86,11 +61,17 @@ typedef struct { int32_t status; // query status TSKEY lastKey; // the lastKey value before query executed STimeWindow w; // whole query time window - STimeWindow curWindow; // current query window int32_t windowIndex; // index of active time window result for interval query STSCursor cur; } SQueryStatusInfo; +typedef struct { + SArray *dataBlockInfos; + int64_t firstSeekTimeUs; + int64_t numOfRowsInMemTable; + char *result; +} STableBlockDist; + #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = rand(); @@ -127,12 +108,13 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) #define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) -#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) +#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) -static void setQueryStatus(SQuery *pQuery, int8_t status); static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv); -#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) +int32_t getMaximumIdleDurationSec() { + return tsShellActivityTimer * 2; +} static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -171,29 +153,19 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { tw->ekey -= 1; } -#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) -#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) - -// todo move to utility -static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); - static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); static void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); -static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId); + SDataStatis *pStatis, SExprInfo* pExprInfo); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); static void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static void buildTagQueryResult(SQInfo *pQInfo); -static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); -static int32_t checkForQueryBuf(size_t numOfTables); +static int32_t setTimestampListJoinInfo(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); @@ -212,22 +184,22 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) { bool isnull = isNull(pElem, pFilterInfo->info.type); if (isnull) { - if (pFilterElem->fp == isNull_filter) { + if (pFilterElem->fp == isNullOperator) { qualified = true; break; } else { continue; } } else { - if (pFilterElem->fp == notNull_filter) { + if (pFilterElem->fp == notNullOperator) { qualified = true; break; - } else if (pFilterElem->fp == isNull_filter) { + } else if (pFilterElem->fp == isNullOperator) { continue; } } - if (pFilterElem->fp(pFilterElem, pElem, pElem)) { + if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo->info.type)) { qualified = true; break; } @@ -288,12 +260,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static int32_t getMergeResultGroupId(int32_t groupIndex) { - int32_t base = 50000000; - return base + (groupIndex * 10000); -} - -bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { +bool isGroupbyColumn(SSqlGroupbyExpr *pGroupbyExpr) { if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { return false; } @@ -313,6 +280,17 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { return false; } +bool isStabledev(SQuery* pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functId = pQuery->pExpr1[i].base.functionId; + if (functId == TSDB_FUNC_STDDEV_DST) { + return true; + } + } + + return false; +} + int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { assert(pGroupbyExpr != NULL); @@ -337,7 +315,7 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { return type; } -bool isSelectivityWithTagsQuery(SQuery *pQuery) { +static bool isSelectivityWithTagsQuery(SQuery *pQuery) { bool hasTags = false; int32_t numOfSelectivity = 0; @@ -348,7 +326,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { continue; } - if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + if ((aAggs[functId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { numOfSelectivity++; } } @@ -360,7 +338,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { return false; } -bool isProjQuery(SQuery *pQuery) { +static bool isProjQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functId = pQuery->pExpr1[i].base.functionId; if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) { @@ -371,17 +349,14 @@ bool isProjQuery(SQuery *pQuery) { return true; } -bool isTSCompQuery(SQuery *pQuery) { return pQuery->pExpr1[0].base.functionId == TSDB_FUNC_TS_COMP; } - -static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery* pQuery = pRuntimeEnv->pQuery; +static bool isTsCompQuery(SQuery *pQuery) { return pQuery->pExpr1[0].base.functionId == TSDB_FUNC_TS_COMP; } +static bool limitOperator(SQuery* pQuery, void* qinfo) { if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, - pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + qinfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); assert(pQuery->rec.rows >= 0); setQueryStatus(pQuery, QUERY_COMPLETED); return true; @@ -408,7 +383,7 @@ static bool isTopBottomQuery(SQuery *pQuery) { static bool timeWindowInterpoRequired(SQuery *pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TWA) { + if (functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_INTERP) { return true; } } @@ -461,60 +436,84 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, char *pData, +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { + bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - int32_t *p1 = - (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (p1 != NULL) { - pWindowResInfo->curIndex = *p1; + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; + } + + if (p1 != NULL) { + for(int32_t i = pResultRowInfo->size - 1; i >= 0; --i) { + if (pResultRowInfo->pResult[i] == (*p1)) { + pResultRowInfo->curIndex = i; + existed = true; + break; + } + } + } } else { - if (!masterscan) { // not master scan, do not add new timewindow - return NULL; + if (p1 != NULL) { // group by column query + return *p1; } + } + if (!existed) { // TODO refactor // more than the capacity, reallocate the resources - if (pWindowResInfo->size >= pWindowResInfo->capacity) { + if (pResultRowInfo->size >= pResultRowInfo->capacity) { int64_t newCapacity = 0; - if (pWindowResInfo->capacity > 10000) { - newCapacity = (int64_t)(pWindowResInfo->capacity * 1.25); + if (pResultRowInfo->capacity > 10000) { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); } else { - newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5); + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); } - char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pWindowResInfo->pResult = (SResultRow **)t; + pResultRowInfo->pResult = (SResultRow **)t; - int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc); + int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; + memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); - pWindowResInfo->capacity = (int32_t)newCapacity; + pResultRowInfo->capacity = (int32_t)newCapacity; } - SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool); - pWindowResInfo->pResult[pWindowResInfo->size] = pResult; - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + SResultRow *pResult = NULL; + + if (p1 == NULL) { + pResult = getNewResultRow(pRuntimeEnv->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + } else { + pResult = *p1; } - // add a new result set for a new group - pWindowResInfo->curIndex = pWindowResInfo->size++; - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), - (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + pResultRowInfo->pResult[pResultRowInfo->size] = pResult; + pResultRowInfo->curIndex = pResultRowInfo->size++; } // too many time window in query - if (pWindowResInfo->size > MAX_INTERVAL_TIME_WINDOW) { + if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return getResultRow(pWindowResInfo, pWindowResInfo->curIndex); + return getResultRow(pResultRowInfo, pResultRowInfo->curIndex); } // get the correct time window according to the handled timestamp @@ -524,7 +523,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t t if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value w.skey = pWindowResInfo->prevSKey; if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision); + w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; } else { w.ekey = w.skey + pQuery->interval.interval - 1; } @@ -588,7 +587,6 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf if (pData->num >= numOfRowsPerPage) { // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer @@ -611,26 +609,20 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, - STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) { +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t groupId) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - // todo refactor - int64_t uid = getResultInfoUId(pRuntimeEnv); - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, uid); + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); if (pResultRow == NULL) { - *newWind = false; - - // no master scan, no result generated means error occurs - return masterscan? -1:0; + *pResult = NULL; + return TSDB_CODE_SUCCESS; } - *newWind = true; - // not assign result buffer yet, add new result buffer if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -700,86 +692,65 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -/** - * NOTE: the query status only set for the first scan of master scan. - */ -static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->scanFlag != MASTER_SCAN) { - return pWindowResInfo->size; - } - - // for group by normal column query, close time window and return. - if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { - closeAllTimeWindow(pWindowResInfo); - return pWindowResInfo->size; - } +static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { + int64_t skey = TSKEY_INITIAL_VAL; + int32_t i = 0; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } - // no qualified results exist, abort check - int32_t numOfClosed = 0; + // new closed result rows + if (timeWindowInterpo) { + if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { + if (i > 0) { // the first time window, the startInterp is false. + assert(pResult->startInterp); + } - if (pWindowResInfo->size == 0) { - return pWindowResInfo->size; + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } + } else { + if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } + } } - // query completed - if ((lastKey >= pQuery->current->win.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (lastKey <= pQuery->current->win.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - closeAllTimeWindow(pWindowResInfo); - - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); - } else { // set the current index to be the last unclosed window - int32_t i = 0; - int64_t skey = TSKEY_INITIAL_VAL; + // all result rows are closed, set the last one to be the skey + if (skey == TSKEY_INITIAL_VAL) { + pResultRowInfo->curIndex = pResultRowInfo->size - 1; + } else { - for (i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pResult = pWindowResInfo->pResult[i]; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; if (pResult->closed) { - numOfClosed += 1; - continue; - } - - TSKEY ekey = pResult->win.ekey; - if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->win.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { - closeTimeWindow(pWindowResInfo, i); - } else { - skey = pResult->win.skey; break; } } - // all windows are closed, set the last one to be the skey - if (skey == TSKEY_INITIAL_VAL) { - assert(i == pWindowResInfo->size); - pWindowResInfo->curIndex = pWindowResInfo->size - 1; + if (i == pResultRowInfo->size - 1) { + pResultRowInfo->curIndex = i; } else { - pWindowResInfo->curIndex = i; + pResultRowInfo->curIndex = i + 1; // current not closed result object } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; - - // the number of completed slots are larger than the threshold, return current generated results to client. - if (numOfClosed > pQuery->rec.threshold) { - qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", - GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - } else { - qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, - numOfClosed); - } + pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; } +} - // output has reached the limitation, set query completed - if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && - pRuntimeEnv->scanFlag == MASTER_SCAN) { - setQueryStatus(pQuery, QUERY_COMPLETED); +static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery, bool timeWindowInterpo) { + if ((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey && (!ascQuery))) { + closeAllResultRows(pResultRowInfo); + pResultRowInfo->curIndex = pResultRowInfo->size - 1; + } else { + int32_t step = ascQuery? 1:-1; + doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey - step, ascQuery, timeWindowInterpo); } - - assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); - return numOfClosed; } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, @@ -822,52 +793,57 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, - int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { +static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock); + +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset, + int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal, SArray *pDataBlock) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool hasPrev = pCtx[0].preAggVals.isSet; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; - pCtx[k].size = forwardStep; - pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].size = forwardStep; + pCtx[k].startTs = pWin->skey; - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; - } + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, numOfTotal, pDataBlock); - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } + int32_t pos = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); + if (dataBlock != NULL) { + pCtx[k].pInput = (char *)dataBlock + pos * pCtx[k].inputBytes; + } - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunction(&pCtx[k]); - } + if (tsCol != NULL) { + pCtx[k].ptsList = &tsCol[pos]; + } + + int32_t functionId = pQuery->pExpr1[k].base.functionId; + + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { + pCtx[k].preAggVals.isSet = false; + } - // restore it - pCtx[k].preAggVals.isSet = hasPrev; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); } + + // restore it + pCtx[k].preAggVals.isSet = hasPrev; } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { - SQuery * pQuery = pRuntimeEnv->pQuery; +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset) { + SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].startTs = pWin->skey; - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); } } } @@ -979,8 +955,8 @@ static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { return NULL; } -static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, - SArray *pDataBlock) { +// todo refactor +static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { if (pDataBlock == NULL) { return NULL; } @@ -991,15 +967,9 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas int32_t functionId = pQuery->pExpr1[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - - sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); - sas->colList = pQuery->colList; - sas->numOfCols = pQuery->numOfCols; - sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); - - if (sas->data == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); + sas->colList = pQuery->colList; + sas->numOfCols = pQuery->numOfCols; // here the pQuery->colList and sas->colList are identical int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); @@ -1047,26 +1017,9 @@ static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, in } } -//static double getTSWindowInterpoVal(SColumnInfoData* pColInfo, int16_t srcColIndex, int16_t rowIndex, TSKEY key, char** prevRow, TSKEY* tsCols, int32_t step) { -// TSKEY start = tsCols[rowIndex]; -// TSKEY prevTs = (rowIndex == 0)? *(TSKEY *) prevRow[0] : tsCols[rowIndex - step]; -// -// double v1 = 0, v2 = 0, v = 0; -// char *prevVal = (rowIndex == 0)? prevRow[srcColIndex] : ((char*)pColInfo->pData) + (rowIndex - step) * pColInfo->info.bytes; -// -// GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal); -// GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + rowIndex * pColInfo->info.bytes); -// -// SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; -// SPoint point2 = (SPoint){.key = start, .val = &v2}; -// SPoint point = (SPoint){.key = key, .val = &v}; -// taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); -// -// return v; -//} - // window start key interpolation -static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { +static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, + SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win, int16_t type) { SQuery* pQuery = pRuntimeEnv->pQuery; TSKEY curTs = tsCols[pos]; @@ -1120,13 +1073,13 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32 return true; } -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) { +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, + int32_t rowIndex) { if (pDataBlock == NULL) { return; } SQuery* pQuery = pRuntimeEnv->pQuery; - int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); @@ -1146,6 +1099,53 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY return ts; } +static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray *pDataBlock, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + if (!pRuntimeEnv->timeWindowInterpo) { + return; + } + + assert(pDataBlock != NULL); + + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t fillType = pQuery->fillType; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); + + TSKEY *tsCols = (TSKEY *)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, win, fillType); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } + + // point interpolation does not require the end key time window interpolation. + if (isPointInterpoQuery(pQuery)) { + return; + } + + // interpolation query does not generate the time window end interpolation + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + } +} + /** * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions * @param pRuntimeEnv @@ -1161,22 +1161,18 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + TSKEY *tsCols = NULL; if (pDataBlock != NULL) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); tsCols = (TSKEY *)(pColInfo->pData); } - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pQuery->pExpr1[k]); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -1186,76 +1182,50 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { - tfree(sasArray); - return; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + goto _end; } int32_t forwardStep = 0; int32_t startPos = pQuery->pos; - // in case of repeat scan/reverse scan, no new time window added. - if (hasTimeWindow) { - TSKEY ekey = reviseWindowEkey(pQuery, &win); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); + TSKEY ekey = reviseWindowEkey(pQuery, &win); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - // prev time window not interpolation yet. - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { - for(int32_t j = prevIndex; j < curIndex; ++j) { - SResultRow *pRes = pWindowResInfo->pResult[j]; - - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows); + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { + for(int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow *pRes = pWindowResInfo->pResult[j]; + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; } - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - assert (ret == TSDB_CODE_SUCCESS); // null data, too many state code - } - - // window start key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = pQuery->pos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &w, masterScan, &pResult, groupId); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step; + int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } + doBlockwiseApplyFunctions(pRuntimeEnv, &w, startPos, 0, tsCols, pDataBlockInfo->rows, pDataBlock); } - bool pStatus = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + assert (ret == TSDB_CODE_SUCCESS); } - int32_t index = pWindowResInfo->curIndex; - STimeWindow nextWin = win; + // window start key interpolation + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &win, pQuery->pos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows, pDataBlock); + STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos); @@ -1264,46 +1234,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (!hasTimeWindow) { - continue; - } - - TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); + ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); // window start(end) key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } - - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1)*step; - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } - } - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &nextWin, startPos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows, pDataBlock); } - pWindowResInfo->curIndex = index; } else { /* * the sqlfunctionCtx parameters should be set done before all functions are invoked, @@ -1313,34 +1256,20 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pExpr1[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - pCtx[k].nStartQueryTimestamp = pDataBlockInfo->window.skey; + pCtx[k].startTs = pQuery->window.skey; aAggs[functionId].xFunction(&pCtx[k]); } } } + _end: if (pRuntimeEnv->timeWindowInterpo) { - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock); - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); + int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, rowIndex); } - - tfree(sasArray); } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - if (isNull(pData, type)) { // ignore the null value - return -1; - } - - int32_t GROUPRESULTID = 1; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; // not assign result buffer yet, add new result buffer, TODO remove it @@ -1352,25 +1281,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); - - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - uint64_t uid = groupIndex; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); - if (pResultRow == NULL) { return -1; } - int64_t v = -1; - switch(type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(pData); break; - case TSDB_DATA_TYPE_SMALLINT: v = GET_INT16_VAL(pData); break; - case TSDB_DATA_TYPE_INT: v = GET_INT32_VAL(pData); break; - case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break; - } + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->resultRowInfo, d, len, true, groupIndex); + assert (pResultRow != NULL); + int64_t v = -1; + GET_TYPED_DATA(v, int64_t, type, pData); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (pResultRow->key == NULL) { pResultRow->key = malloc(varDataTLen(pData)); @@ -1384,7 +1302,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; } @@ -1439,7 +1357,7 @@ static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQuery *pQuery = pRuntimeEnv->pQuery; - STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first @@ -1447,25 +1365,25 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { return TS_JOIN_TAG_NOT_EQUALS; } - TSKEY key = *(TSKEY *)((char*)pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); + TSKEY key = *(TSKEY *)((char*)pCtx[0].pInput + TSDB_KEYSIZE * offset); #if defined(_DEBUG_VIEW) printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag.i64Key, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, - pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); + elem.ts, key, elem.tag.i64, pQuery->order.order, pRuntimeEnv->pTsBuf->tsOrder, + pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); #endif if (QUERY_IS_ASC_QUERY(pQuery)) { if (key < elem.ts) { return TS_JOIN_TS_NOT_EQUALS; } else if (key > elem.ts) { - assert(false); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); } } else { if (key > elem.ts) { return TS_JOIN_TS_NOT_EQUALS; } else if (key < elem.ts) { - assert(false); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); } } @@ -1491,7 +1409,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx // denote the order type if ((functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST)) { - return pCtx->param[0].i64Key == pQuery->order.order; + return pCtx->param[0].i64 == pQuery->order.order; } // in the supplementary scan, only the following functions need to be executed @@ -1502,24 +1420,26 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return true; } -void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SQuery* pQuery = pRuntimeEnv->pQuery; +void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv *pRuntimeEnv, SArray *pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, + int32_t type) { + SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionId != TSDB_FUNC_TWA) { + if (functionId != TSDB_FUNC_TWA && functionId != TSDB_FUNC_INTERP) { pRuntimeEnv->pCtx[k].start.key = INT64_MIN; continue; } - SColIndex* pColIndex = &pQuery->pExpr1[k].base.colInfo; - int16_t index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + SColIndex * pColIndex = &pQuery->pExpr1[k].base.colInfo; + int16_t index = pColIndex->colIndex; + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]); + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); } else { GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } @@ -1529,14 +1449,25 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; SPoint point2 = (SPoint){.key = curTs, .val = &v2}; SPoint point = (SPoint){.key = windowKey, .val = &v}; - taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); - if (type == RESULT_ROW_START_INTERP) { - pRuntimeEnv->pCtx[k].start.key = point.key; - pRuntimeEnv->pCtx[k].start.val = v; + if (functionId == TSDB_FUNC_TWA) { + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pRuntimeEnv->pCtx[k].start.key = point.key; + pRuntimeEnv->pCtx[k].start.val = v; + } else { + pRuntimeEnv->pCtx[k].end.key = point.key; + pRuntimeEnv->pCtx[k].end.val = v; + } } else { - pRuntimeEnv->pCtx[k].end.key = point.key; - pRuntimeEnv->pCtx[k].end.val = v; + if (type == RESULT_ROW_START_INTERP) { + pRuntimeEnv->pCtx[k].start.key = prevTs; + pRuntimeEnv->pCtx[k].start.val = v1; + + pRuntimeEnv->pCtx[k].end.key = curTs; + pRuntimeEnv->pCtx[k].end.val = v2; + } } } } @@ -1579,22 +1510,19 @@ static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pData } static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* item = pQuery->current; + int64_t groupId = item->groupIndex; + SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0); TSKEY *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL; - bool groupbyColumnValue = pRuntimeEnv->groupbyNormalCol; - - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + bool groupbyColumnValue = pRuntimeEnv->groupbyColumn; int16_t type = 0; int16_t bytes = 0; @@ -1606,8 +1534,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pQuery->pExpr1[k]); pCtx[k].size = 1; } @@ -1622,9 +1550,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // from top to bottom in desc // from bottom to top in asc order - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { qDebug("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, - pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); + pQuery->order.order, pRuntimeEnv->pTsBuf->cur.order); } int32_t offset = -1; @@ -1634,7 +1562,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { offset = GET_COL_DATA_POS(pQuery, j, step); - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { int32_t ret = doTSJoinFilter(pRuntimeEnv, offset); if (ret == TS_JOIN_TAG_NOT_EQUALS) { break; @@ -1652,19 +1580,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); - int64_t ts = tsCols[offset]; + int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; - } - - if (!hasTimeWindow) { - continue; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { // null data, too many state code + goto _end; } // window start key interpolation @@ -1674,19 +1597,20 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { for (int32_t k = prevWindowIndex; k < curIndex; ++k) { SResultRow *pRes = pWindowResInfo->pResult[k]; + if (pRes->closed) { + assert(resultRowInterpolated(pResult, RESULT_ROW_START_INTERP) && resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + continue; + } - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &pRes->win, masterScan, &pResult, groupId); assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &pRes->win, offset); } // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, - &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1695,12 +1619,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); } - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &win, offset); + int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; - int32_t index = pWindowResInfo->curIndex; - while (1) { getNextTimeWindow(pQuery, &nextWin); if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || @@ -1713,27 +1635,57 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (hasTimeWindow) { - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); - closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); - } + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); + doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset); } + // restore the index, add the result row will move the index pWindowResInfo->curIndex = index; } else { // other queries // decide which group this rows belongs to according to current state value + char* val = NULL; if (groupbyColumnValue) { - char *val = groupbyColumnData + bytes * offset; + val = groupbyColumnData + bytes * offset; + if (isNull(val, type)) { // ignore the null value + continue; + } int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + } + + if (pRuntimeEnv->stabledev) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if (functionId != TSDB_FUNC_STDDEV_DST) { + continue; + } + + pRuntimeEnv->pCtx[k].param[0].arr = NULL; + pRuntimeEnv->pCtx[k].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + // todo opt perf + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t i = 0; i < numOfGroup; ++i) { + SInterResult *p = taosArrayGet(pRuntimeEnv->prevResult, i); + if (memcmp(p->tags, val, bytes) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t f = 0; f < numOfCols; ++f) { + SStddevInterResult *pres = taosArrayGet(p->pResult, f); + if (pres->colId == pQuery->pExpr1[k].base.colInfo.colId) { + pRuntimeEnv->pCtx[k].param[0].arr = pres->pResult; + break; + } + } + } + } } } @@ -1748,64 +1700,59 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS prevTs = tsCols[offset]; prevRowIndex = offset; - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { // if timestamp filter list is empty, quit current query - if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { setQueryStatus(pQuery, QUERY_COMPLETED); break; } } } - assert(offset >= 0); - if (tsCols != NULL) { - item->lastKey = tsCols[offset] + step; - } else { - item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step; + _end: + assert(offset >= 0 && tsCols != NULL); + if (prevTs != INT64_MIN && prevTs != *(int64_t*)pRuntimeEnv->prevRow[0]) { + assert(prevRowIndex >= 0); + item->lastKey = prevTs + step; } - if (pRuntimeEnv->pTSBuf != NULL) { - item->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + // In case of all rows in current block are not qualified + if (pRuntimeEnv->timeWindowInterpo && prevRowIndex != -1) { + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, prevRowIndex); } - // todo refactor: extract method - for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); + if (pRuntimeEnv->pTsBuf != NULL) { + item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); } - - free(sasArray); } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQInfo = pQuery->current; - SResultRowInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + STableQueryInfo* pTableQueryInfo = pQuery->current; + SResultRowInfo* pResultRowInfo = &pRuntimeEnv->resultRowInfo; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } - // update the lastkey of current table + // update the lastkey of current table for projection/aggregation query TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // interval query with limit applied int32_t numOfRes = 0; - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { - numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); - } else { - numOfRes = (int32_t)getNumOfResult(pRuntimeEnv); + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { + numOfRes = pResultRowInfo->size; + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); + } else { // projection query + numOfRes = (int32_t) getNumOfResult(pRuntimeEnv); // update the number of output result - if (numOfRes > 0 && pQuery->checkBuffer == 1) { + if (numOfRes > 0 && pQuery->checkResultBuf == 1) { assert(numOfRes >= pQuery->rec.rows); pQuery->rec.rows = numOfRes; @@ -1817,8 +1764,8 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl setQueryStatus(pQuery, QUERY_COMPLETED); } - if (((pTableQInfo->lastKey > pTableQInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pTableQInfo->lastKey < pTableQInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { + if (((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { setQueryStatus(pQuery, QUERY_COMPLETED); } } @@ -1828,14 +1775,13 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl } void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId) { + SDataStatis *pStatis, SExprInfo* pExprInfo) { - int32_t functionId = pQuery->pExpr1[colIndex].base.functionId; - int32_t colId = pQuery->pExpr1[colIndex].base.colInfo.colId; + int32_t functionId = pExprInfo->base.functionId; + int32_t colId = pExprInfo->base.colInfo.colId; SDataStatis *tpField = NULL; - pCtx->hasNull = hasNullValue(&pQuery->pExpr1[colIndex].base.colInfo, pStatis, &tpField); - pCtx->aInputElemBuf = inputData; + pCtx->hasNull = hasNullValue(&pExprInfo->base.colInfo, pStatis, &tpField); if (tpField != NULL) { pCtx->preAggVals.isSet = true; @@ -1850,79 +1796,24 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY // limit/offset query will affect this value pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; - // minimum value no matter ascending/descending order query - pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1); - assert(pCtx->startOffset >= 0); + // set the start position in current block + int32_t offset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1); + if (inputData != NULL) { + pCtx->pInput = (char*)inputData + offset * pCtx->inputBytes; + } - uint32_t status = aAggs[functionId].nStatus; + uint32_t status = aAggs[functionId].status; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = &tsCol[pCtx->startOffset]; + pCtx->ptsList = tsCol + offset; } - if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { - // last_dist or first_dist function - // store the first&last timestamp into the intermediate buffer [1], the true - // value may be null but timestamp will never be null - } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || - functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { - /* - * least squares function needs two columns of input, currently, the x value of linear equation is set to - * timestamp column, and the y-value is the column specified in pQuery->pExpr1[i].colIdxInBuffer - * - * top/bottom function needs timestamp to indicate when the - * top/bottom values emerge, so does diff function - */ - if (functionId == TSDB_FUNC_TWA) { - pCtx->param[1].i64Key = pQuery->window.skey; - pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[2].i64Key = pQuery->window.ekey; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - } - - } else if (functionId == TSDB_FUNC_ARITHM) { - pCtx->param[1].pz = param; - } else if (functionId == TSDB_FUNC_SPREAD) { // set the statistics data for primary time stamp column + if (functionId == TSDB_FUNC_SPREAD) { // set the statistics data for primary time stamp column if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { pCtx->preAggVals.isSet = true; pCtx->preAggVals.statis.min = pBlockInfo->window.skey; pCtx->preAggVals.statis.max = pBlockInfo->window.ekey; } - } else if (functionId == TSDB_FUNC_INTERP) { - SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); - - SInterpInfoDetail *pInterpInfo = (SInterpInfoDetail *)GET_ROWCELL_INTERBUF(pInfo); - pInterpInfo->type = (int8_t)pQuery->fillType; - pInterpInfo->ts = pQuery->window.skey; - pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - - if (pQuery->fillVal != NULL) { - if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) { - pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; - } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value - if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[1], (char*) &pQuery->fillVal[colIndex], pCtx->inputBytes, pCtx->inputType); - } - } - } - } else if (functionId == TSDB_FUNC_TS_COMP) { - pCtx->param[0].i64Key = vgId; - pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; } - -#if defined(_DEBUG_VIEW) - // int64_t *tsList = (int64_t *)primaryColumnData; -// int64_t s = tsList[0]; -// int64_t e = tsList[size - 1]; - -// if (IS_DATA_BLOCK_LOADED(blockStatus)) { -// qDebug("QInfo:%p query ts:%lld-%lld, offset:%d, rows:%d, bstatus:%d, -// functId:%d", GET_QINFO_ADDR(pQuery), -// s, e, startOffset, size, blockStatus, functionId); -// } else { -// qDebug("QInfo:%p block not loaded, bstatus:%d", -// GET_QINFO_ADDR(pQuery), blockStatus); -// } -#endif } // set the output buffer for the selectivity + tag query @@ -1945,7 +1836,7 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; - } else if ((aAggs[pSqlFuncMsg->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + } else if ((aAggs[pSqlFuncMsg->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { p = &pCtx[i]; } else if (pSqlFuncMsg->functionId == TSDB_FUNC_TS || pSqlFuncMsg->functionId == TSDB_FUNC_TAG) { // tag function may be the group by tag column @@ -1967,24 +1858,41 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return TSDB_CODE_SUCCESS; } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, int16_t order, int32_t vgId) { qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); - pRuntimeEnv->offset = calloc(pQuery->numOfOutput, sizeof(int16_t)); + pRuntimeEnv->interBufSize = getOutputInterResultBufSize(pQuery); + pRuntimeEnv->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); + + pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pRuntimeEnv->keyBuf = malloc(pQuery->maxSrcColumnSize + sizeof(int64_t)); + pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); + pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQuery->numOfCols + pQuery->srcRowSize); + pRuntimeEnv->tagVal = malloc(pQuery->tagLen); + pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); + pRuntimeEnv->offset = calloc(pQuery->numOfOutput, sizeof(int16_t)); pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->sasArray = calloc(pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { + if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL || + pRuntimeEnv->sasArray == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || + pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { goto _clean; } + char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pRuntimeEnv->prevRow; + pRuntimeEnv->prevRow[0] = start; + for(int32_t i = 1; i < pQuery->numOfCols; ++i) { + pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQuery->colList[i-1].bytes; + } + pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pExpr1[i].base; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - SColIndex* pIndex = &pSqlFuncMsg->colInfo; + SColIndex * pIndex = &pSqlFuncMsg->colInfo; if (TSDB_COL_REQ_NULL(pIndex->flag)) { pCtx->requireNull = true; @@ -1996,8 +1904,12 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int32_t index = pSqlFuncMsg->colInfo.colIndex; if (TSDB_COL_IS_TAG(pIndex->flag)) { if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor - SSchema s = tGetTableNameColumnSchema(); + SSchema *s = tGetTbnameColumnSchema(); + pCtx->inputBytes = s->bytes; + pCtx->inputType = s->type; + } else if (pIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SSchema s = tGetBlockDistColumnSchema(); pCtx->inputBytes = s.bytes; pCtx->inputType = s.type; } else { @@ -2029,7 +1941,11 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order for (int32_t j = 0; j < pCtx->numOfParams; ++j) { int16_t type = pSqlFuncMsg->arg[j].argType; int16_t bytes = pSqlFuncMsg->arg[j].argBytes; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (pSqlFuncMsg->functionId == TSDB_FUNC_STDDEV_DST) { + continue; + } + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); @@ -2043,26 +1959,52 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int32_t f = pQuery->pExpr1[0].base.functionId; assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); - pCtx->param[2].i64Key = order; + pCtx->param[2].i64 = order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[3].i64Key = functionId; + pCtx->param[3].i64 = functionId; pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[1].i64Key = pQuery->order.orderColId; + pCtx->param[1].i64 = pQuery->order.orderColId; + } else if (functionId == TSDB_FUNC_INTERP) { + pCtx->param[2].i64 = (int8_t)pQuery->fillType; + if (pQuery->fillVal != NULL) { + if (isNull((const char *)&pQuery->fillVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value + if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->fillVal[i], pCtx->inputBytes, pCtx->inputType); + } + } + } + } else if (functionId == TSDB_FUNC_TS_COMP) { + pCtx->param[0].i64 = vgId; + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == TSDB_FUNC_TWA) { + pCtx->param[1].i64 = pQuery->window.skey; + pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[2].i64 = pQuery->window.ekey; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == TSDB_FUNC_ARITHM) { + pRuntimeEnv->sasArray[i].data = calloc(pQuery->numOfCols, POINTER_BYTES); + if (pRuntimeEnv->sasArray[i].data == NULL) { + goto _clean; + } + + pCtx->param[1].pz = (char*) &pRuntimeEnv->sasArray[i]; } if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; - pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes; + pRuntimeEnv->rowCellInfoOffset[i] = + pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes; } - } *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; // if it is group by normal column, do not set output buffer, the output buffer is pResult // fixed output query/multi-output query for normal table - if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!pRuntimeEnv->groupbyColumn && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { resetDefaultResInfoOutputBuf(pRuntimeEnv); } @@ -2077,6 +2019,11 @@ _clean: tfree(pRuntimeEnv->pCtx); tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->rowCellInfoOffset); + tfree(pRuntimeEnv->sasArray); + tfree(pRuntimeEnv->pResultRowHashTable); + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2094,6 +2041,7 @@ static void doFreeQueryHandle(SQInfo* pQInfo) { assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL); } + static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { if (pRuntimeEnv->pQuery == NULL) { return; @@ -2103,7 +2051,17 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); qDebug("QInfo:%p teardown runtime env", pQInfo); - cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo); + cleanupResultRowInfo(&pRuntimeEnv->resultRowInfo); + + if (isTsCompQuery(pQuery)) { + FILE *f = *(FILE **)pQuery->sdata[0]->data; + + if (f) { + fclose(f); + *(FILE **)pQuery->sdata[0]->data = NULL; + } + } + if (pRuntimeEnv->pCtx != NULL) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -2120,27 +2078,59 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->pCtx); } + if (pRuntimeEnv->sasArray != NULL) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + tfree(pRuntimeEnv->sasArray[i].data); + } + + tfree(pRuntimeEnv->sasArray); + } + pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pQInfo); - pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); + pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->keyBuf); tfree(pRuntimeEnv->rowCellInfoOffset); tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); taosHashCleanup(pRuntimeEnv->pResultRowHashTable); pRuntimeEnv->pResultRowHashTable = NULL; pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); + taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); + pRuntimeEnv->prevResult = NULL; +} + +static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { + return pQInfo->rspContext != NULL; } -#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) +bool isQueryKilled(SQInfo *pQInfo) { + if (IS_QUERY_KILLED(pQInfo)) { + return true; + } + + // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived + // abort current query execution. + if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs) > getMaximumIdleDurationSec()) && + (!needBuildResAfterQueryComplete(pQInfo))) { + + assert(pQInfo->startExecTs != 0); + qDebug("QInfo:%p retrieve not arrive beyond %d sec, abort current query execution, start:%"PRId64", current:%d", pQInfo, 1, + pQInfo->startExecTs, taosGetTimestampSec()); + return true; + } + + return false; +} -static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} +void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { SQuery* pQuery = pRuntimeEnv->pQuery; @@ -2149,7 +2139,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { } // Note:top/bottom query is fixed output query - if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyColumn) { return true; } @@ -2166,7 +2156,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { continue; } - if (!IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus)) { + if (!IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].status)) { return true; } } @@ -2177,8 +2167,8 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { // todo refactor with isLastRowQuery bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionID = pQuery->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_INTERP) { + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_INTERP) { return true; } } @@ -2241,7 +2231,7 @@ static bool needReverseScan(SQuery *pQuery) { * The following 4 kinds of query are treated as the tags query * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query */ -static bool onlyQueryTags(SQuery* pQuery) { +bool onlyQueryTags(SQuery* pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQuery->pExpr1[i]; @@ -2280,9 +2270,9 @@ void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int6 static void setScanLimitationByResultBuffer(SQuery *pQuery) { if (isTopBottomQuery(pQuery)) { - pQuery->checkBuffer = 0; - } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - pQuery->checkBuffer = 0; + pQuery->checkResultBuf = 0; + } else if (isGroupbyColumn(pQuery->pGroupbyExpr)) { + pQuery->checkResultBuf = 0; } else { bool hasMultioutput = false; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -2291,13 +2281,13 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { continue; } - hasMultioutput = IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus); + hasMultioutput = IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].status); if (!hasMultioutput) { break; } } - pQuery->checkBuffer = hasMultioutput ? 1 : 0; + pQuery->checkResultBuf = hasMultioutput ? 1 : 0; } } @@ -2339,7 +2329,6 @@ static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSD static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } -// todo refactor, add iterator static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) { size_t t = taosArrayGetSize(pQInfo->tableGroupInfo.pGroupList); for(int32_t i = 0; i < t; ++i) { @@ -2377,7 +2366,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo return; } - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) { + if (isGroupbyColumn(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) { pQuery->order.order = TSDB_ORDER_ASC; if (pQuery->window.skey > pQuery->window.ekey) { SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2454,7 +2443,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { int32_t num = 0; - if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + if (isGroupbyColumn(pQuery->pGroupbyExpr)) { num = 128; } else if (QUERY_IS_INTERVAL_QUERY(pQuery)) { // time window query, allocate one page for each table size_t s = pQInfo->tableqinfoGroupInfo.numOfTables; @@ -2471,7 +2460,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i SQuery* pQuery = pRuntimeEnv->pQuery; int32_t MIN_ROWS_PER_PAGE = 4; - *rowsize = (int32_t)(pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); + *rowsize = (int32_t)(pQuery->resultRowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); int32_t overhead = sizeof(tFilePage); // one page contains at least two rows @@ -2520,7 +2509,7 @@ static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDat // if isNULL query exists, load the null data column for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; - if (pFilterElem->fp == isNull_filter) { + if (pFilterElem->fp == isNullOperator) { return true; } } @@ -2535,13 +2524,13 @@ static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDat float maxval = (float)(*(double *)(&pDataBlockst->max)); for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { - if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval)) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT)) { return true; } } } else { for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { - if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max)) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type)) { return true; } } @@ -2610,14 +2599,16 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { } int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; - *status = BLK_DATA_NO_NEEDED; + SQueryCostInfo* pCost = &pRuntimeEnv->summary; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf > 0) { + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) { *status = BLK_DATA_ALL_NEEDED; } else { // check if this data block is required to load - // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { @@ -2628,16 +2619,13 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - bool hasTimeWindow = false; SResultRow* pResult = NULL; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; - STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } } @@ -2658,35 +2646,34 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW if ((*status) == BLK_DATA_NO_NEEDED) { qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pRuntimeEnv->summary.discardBlocks += 1; + pCost->discardBlocks += 1; } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); - - pRuntimeEnv->summary.loadBlockStatis += 1; + pCost->loadBlockStatis += 1; if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; + pCost->totalCheckedRows += pBlockInfo->rows; } } else { assert((*status) == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter - pRuntimeEnv->summary.loadBlockStatis += 1; + pCost->loadBlockStatis += 1; tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied - pRuntimeEnv->summary.discardBlocks += 1; + pCost->discardBlocks += 1; qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); (*status) = BLK_DATA_DISCARD; } - pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; - pRuntimeEnv->summary.loadBlocks += 1; + pCost->totalCheckedRows += pBlockInfo->rows; + pCost->loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); if (*pDataBlock == NULL) { return terrno; @@ -2744,7 +2731,7 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; + midPos = (numOfRows >> 1u) + firstPos; if (key < keyList[midPos]) { lastPos = midPos - 1; @@ -2759,70 +2746,51 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { return midPos; } -static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capacity) { +static void expandBuffer(SQueryRuntimeEnv* pRuntimeEnv, int32_t newSize, void* qinfo) { SQuery* pQuery = pRuntimeEnv->pQuery; + SResultRec *pRec = &pQuery->rec; - if (capacity < pQuery->rec.capacity) { - return; - } + assert(newSize > 0); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pQuery->pExpr1[i].bytes; - assert(bytes > 0 && capacity > 0); - char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage)); + char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); if (tmp == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { + memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; } - - // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data; } - qDebug("QInfo:%p realloc output buffer to inc output buffer from: %" PRId64 " rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv), - pQuery->rec.capacity, capacity); - - pQuery->rec.capacity = capacity; + pRec->capacity = newSize; + qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, qinfo, newSize, + pRec->capacity, newSize - pRec->rows); } -// TODO merge with enuserOutputBufferSimple -static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { +static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { + if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyColumn && !isFixedOutputQuery(pRuntimeEnv) && !isTsCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; - if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { - int32_t remain = (int32_t)(pRec->capacity - pRec->rows); - int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain)); + int32_t remain = (int32_t)(pRec->capacity - pRec->rows); + if (remain < numOfRows) { + int32_t newSize = (int32_t)(pRec->capacity + (numOfRows - remain)); + expandBuffer(pRuntimeEnv, newSize, GET_QINFO_ADDR(pRuntimeEnv)); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pQuery->pExpr1[i].bytes; - assert(bytes > 0 && newSize > 0); - - char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); - if (tmp == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } else { - memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); - pQuery->sdata[i] = (tFilePage *)tmp; - } // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; + pRuntimeEnv->pCtx[i].pOutput = pQuery->sdata[i]->data + pRec->rows * bytes; int32_t functionId = pQuery->pExpr1[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; } } - - qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), - newSize, pRec->capacity, newSize - pRec->rows); - - pRec->capacity = newSize; } } } @@ -2830,19 +2798,15 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { SQuery* pQuery = pRuntimeEnv->pQuery; - if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { + if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL) { STimeWindow w = TSWINDOW_INITIALIZER; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } @@ -2864,7 +2828,11 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; - if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) { + if (IS_MASTER_SCAN(pRuntimeEnv)) { + pQuery->numOfCheckedBlocks += 1; + } + + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -2872,13 +2840,13 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { doSetInitialTimewindow(pRuntimeEnv, &blockInfo); // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block - ensureOutputBuffer(pRuntimeEnv, &blockInfo); + ensureOutputBuffer(pRuntimeEnv, blockInfo.rows); SDataStatis *pStatis = NULL; SArray * pDataBlock = NULL; uint32_t status = 0; - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pRuntimeEnv->resultRowInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); if (ret != TSDB_CODE_SUCCESS) { break; } @@ -2912,13 +2880,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_COMPLETED); } - if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - closeAllTimeWindow(&pRuntimeEnv->windowResInfo); - pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window - } else { - assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - } + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + closeAllResultRows(&pRuntimeEnv->resultRowInfo); + pRuntimeEnv->resultRowInfo.curIndex = pRuntimeEnv->resultRowInfo.size - 1; // point to the last time window } return 0; @@ -2928,7 +2892,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { * set tag value in SQLFunctionCtx * e.g.,tag information into input buffer */ -static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) { +static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) { tVariantDestroy(tag); if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { @@ -2973,20 +2937,22 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num return NULL; } -void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { +void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable) { SQuery *pQuery = pRuntimeEnv->pQuery; SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SExprInfo *pExprInfo = &pQuery->pExpr1[0]; - if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) { + if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP && pRuntimeEnv->stableQuery) { assert(pExprInfo->base.numOfParams == 1); int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); + doSetTagValueInParam(pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); } else { // set tag value, by which the results are aggregated. + int32_t offset = 0; + memset(pRuntimeEnv->tagVal, 0, pQuery->tagLen); for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SExprInfo* pLocalExprInfo = &pQuery->pExpr1[idx]; @@ -2996,20 +2962,28 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { } // todo use tag column index to optimize performance - doSetTagValueInParam(tsdb, pTable, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag, + doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag, pLocalExprInfo->type, pLocalExprInfo->bytes); + + if (IS_NUMERIC_TYPE(pLocalExprInfo->type) || pLocalExprInfo->type == TSDB_DATA_TYPE_BOOL) { + memcpy(pRuntimeEnv->tagVal + offset, &pRuntimeEnv->pCtx[idx].tag.i64, pLocalExprInfo->bytes); + } else { + memcpy(pRuntimeEnv->tagVal + offset, pRuntimeEnv->pCtx[idx].tag.pz, pRuntimeEnv->pCtx[idx].tag.nLen); + } + + offset += pLocalExprInfo->bytes; } // set the join tag for first column SSqlFuncMsg *pFuncMsg = &pExprInfo->base; - if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTSBuf != NULL && + if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTsBuf != NULL && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { assert(pFuncMsg->numOfParams == 1); int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; SColumnInfo *pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); + doSetTagValueInParam(pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); int16_t tagType = pRuntimeEnv->pCtx[0].tag.nType; if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { @@ -3017,53 +2991,9 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.pz); } else { qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, pQInfo, - pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64Key); - } - } - } -} - -static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (!mergeFlag) { - pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; - pCtx[i].currentStage = FIRST_STAGE_MERGE; - - RESET_RESULT_INFO(pCtx[i].resultInfo); - aAggs[functionId].init(&pCtx[i]); - } - - pCtx[i].hasNull = true; - pCtx[i].nStartQueryTimestamp = timestamp; - pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); - - // in case of tag column, the tag information should be extracted from input buffer - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { - tVariantDestroy(&pCtx[i].tag); - - int32_t type = pCtx[i].outputType; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); - } else { - tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); + pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64); } - - } - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY) { - continue; } - - aAggs[functionId].distMergeFunc(&pCtx[i]); } } @@ -3170,595 +3100,204 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim } } -typedef struct SCompSupporter { - STableQueryInfo **pTableQueryInfo; - int32_t * position; - SQInfo * pQInfo; -} SCompSupporter; +static int32_t doCopyToSData(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType); -int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { - int32_t left = *(int32_t *)pLeft; - int32_t right = *(int32_t *)pRight; +void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + while(pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { + // all results in current group have been returned to client, try next group + if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { + assert(pGroupResInfo->index == 0); + if ((pQInfo->code = mergeIntoGroupResult(&pQInfo->groupResInfo, pQInfo)) != TSDB_CODE_SUCCESS) { + return; + } + } - SCompSupporter * supporter = (SCompSupporter *)param; - SQueryRuntimeEnv *pRuntimeEnv = &supporter->pQInfo->runtimeEnv; + pQuery->rec.rows = doCopyToSData(&pQInfo->runtimeEnv, pGroupResInfo, TSDB_ORDER_ASC); - int32_t leftPos = supporter->position[left]; - int32_t rightPos = supporter->position[right]; + // current data are all dumped to result buffer, clear it + if (!hasRemainData(pGroupResInfo)) { + cleanupGroupResInfo(pGroupResInfo); + if (!incNextGroup(pGroupResInfo)) { + SET_STABLE_QUERY_OVER(pQInfo); + } + } - /* left source is exhausted */ - if (leftPos == -1) { - return 1; + // enough results in data buffer, return + if (pQuery->rec.rows >= pQuery->rec.threshold) { + break; + } } +} - /* right source is exhausted*/ - if (rightPos == -1) { - return -1; +static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { + if (pTableQueryInfo == NULL) { + return; } - SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; - SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); - - char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); - TSKEY leftTimestamp = GET_INT64_VAL(b1); - - SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; - SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); - - char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); - TSKEY rightTimestamp = GET_INT64_VAL(b2); + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - if (leftTimestamp == rightTimestamp) { - return 0; - } + SWITCH_ORDER(pTableQueryInfo->cur.order); + pTableQueryInfo->cur.vgroupIndex = -1; - return leftTimestamp > rightTimestamp ? 1 : -1; + // set the index at the end of time window + pTableQueryInfo->resInfo.curIndex = pTableQueryInfo->resInfo.size - 1; } -int32_t mergeIntoGroupResult(SQInfo *pQInfo) { - int64_t st = taosGetTimestampUs(); - int32_t ret = TSDB_CODE_SUCCESS; - - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); +static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pWindowResInfo, int32_t order) { + SQuery* pQuery = pRuntimeEnv->pQuery; - while (pQInfo->groupIndex < numOfGroups) { - SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); - ret = mergeIntoGroupResultImpl(pQInfo, group); - if (ret < 0) { // not enough disk space to save the data into disk - return -1; + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + bool closed = getResultRowStatus(pWindowResInfo, i); + if (!closed) { + continue; } - pQInfo->groupIndex += 1; - - // this group generates at least one result, return results - if (ret > 0) { - break; - } + SResultRow *pRow = getResultRow(pWindowResInfo, i); - assert(pQInfo->groupResInfo.numOfDataPages == 0); - qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); - } + // open/close the specified query for each group result + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functId = pQuery->pExpr1[j].base.functionId; + SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j); - SGroupResInfo* info = &pQInfo->groupResInfo; - if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) { - SET_STABLE_QUERY_OVER(pQInfo); + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { + pInfo->complete = false; + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + pInfo->complete = true; + } + } } - - int64_t elapsedTime = taosGetTimestampUs() - st; - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, - pQInfo->groupIndex - 1, numOfGroups, elapsedTime); - - pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; - return TSDB_CODE_SUCCESS; } -void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; +void disableFuncInReverseScan(SQInfo *pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; - // all results have been return to client, try next group - if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) { - pGroupResInfo->numOfDataPages = 0; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; + // group by normal columns and interval query on normal table + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { + disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); + } else { // for simple result of table query, + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor + int32_t functId = pQuery->pExpr1[j].base.functionId; - // current results of group has been sent to client, try next group - if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { - return; // failed to save data in the disk - } + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; + if (pCtx->resultInfo == NULL) { + continue; // resultInfo is NULL, means no data checked in previous scan + } - // check if all results has been sent to client - int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { - SET_STABLE_QUERY_OVER(pQInfo); - return; + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { + pCtx->resultInfo->complete = false; + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + pCtx->resultInfo->complete = true; + } } } +} - SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - int32_t id = pQInfo->groupResInfo.groupId; - SIDList list = getDataBufPagesIdList(pResultBuf, id); +static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - int32_t offset = 0; - int32_t numOfCopiedRows = 0; + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); + SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); - size_t size = taosArrayGetSize(list); - assert(size == pGroupResInfo->numOfDataPages); + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - bool done = false; + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle + STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); + pTableKeyInfo->lastKey = pCheckInfo->lastKey; - //TODO add API for release none-dirty pages -// SPageInfo* prev = NULL; + assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + } + } +} - for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { - SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); +void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SWITCH_ORDER(pRuntimeEnv->pCtx[i].order); + } +} - // release previous buffer pages -// if (prev == NULL) { -// prev = pi; -// } else { -// if (prev->pageId != pi->pageId) { -// releaseResBufPageInfo(pResultBuf, prev); -// prev = pi; -// } -// } +int32_t initResultRow(SResultRow *pResultRow) { + pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); + pResultRow->pageId = -1; + pResultRow->rowId = -1; + return TSDB_CODE_SUCCESS; +} - assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); - int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); +void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; - if (numOfRes > pQuery->rec.capacity - offset) { - numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); - pGroupResInfo->rowId += numOfCopiedRows; - done = true; - } else { - numOfCopiedRows = (int32_t)pData->num; + int32_t tid = 0; + int64_t uid = 0; + SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->resultRowInfo, (char *)&tid, sizeof(tid), true, uid); - pGroupResInfo->pageId += 1; - pGroupResInfo->rowId = 0; - } + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + pCtx->pOutput = pQuery->sdata[i]->data; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - char * pDest = pQuery->sdata[i]->data; + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i); + RESET_RESULT_INFO(pCellInfo); + pCtx->resultInfo = pCellInfo; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, - (size_t)bytes * numOfCopiedRows); + // set the timestamp output buffer for top/bottom/diff query + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; } - offset += numOfCopiedRows; - if (done) { - break; - } + memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pExpr1[i].bytes * pQuery->rec.capacity)); } - assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; + initCtxOutputBuf(pRuntimeEnv); } -int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { - SQuery* pQuery = pRuntimeEnv->pQuery; +void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { + SQuery *pQuery = pRuntimeEnv->pQuery; + // reset the execution contexts for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pExpr1[j].base.functionId; + assert(functionId != TSDB_FUNC_DIFF); - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { - continue; + // set next output position + if (IS_OUTER_FORWARD(aAggs[functionId].status)) { + pRuntimeEnv->pCtx[j].pOutput += pRuntimeEnv->pCtx[j].outputBytes * output; } - SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j); - assert(pResultInfo != NULL); - - if (pResultInfo->numOfRes > 0) { - return pResultInfo->numOfRes; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + /* + * NOTE: for top/bottom query, the value of first column of output (timestamp) are assigned + * in the procedure of top/bottom routine + * the output buffer in top/bottom routine is ptsOutputBuf, so we need to forward the output buffer + * + * diff function is handled in multi-output function + */ + pRuntimeEnv->pCtx[j].ptsOutputBuf = (char*)pRuntimeEnv->pCtx[j].ptsOutputBuf + TSDB_KEYSIZE * output; } - } - return 0; + RESET_RESULT_INFO(pRuntimeEnv->pCtx[j].resultInfo); + } } -int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - size_t size = taosArrayGetSize(pGroup); - tFilePage **buffer = pQuery->sdata; - - int32_t *posList = calloc(size, sizeof(int32_t)); - STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); - - if (pTableList == NULL || posList == NULL) { - tfree(posList); - tfree(pTableList); - - qError("QInfo:%p failed alloc memory", pQInfo); - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - // todo opt for the case of one table per group - int32_t numOfTables = 0; - SIDList pageList = NULL; - int32_t tid = -1; - - for (int32_t i = 0; i < size; ++i) { - STableQueryInfo *item = taosArrayGetP(pGroup, i); - - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); - if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables++] = item; - tid = TSDB_TABLEID(item->pTable)->tid; - pageList = list; - } - } - - // there is no data in current group - if (numOfTables == 0) { - tfree(posList); - tfree(pTableList); - return 0; - } else if (numOfTables == 1) { // no need to merge results since only one table in each group - tfree(posList); - tfree(pTableList); - - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); - pGroupResInfo->groupId = tid; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; - - return pGroupResInfo->numOfDataPages; - } - - SCompSupporter cs = {pTableList, posList, pQInfo}; - - SLoserTreeInfo *pTree = NULL; - tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - - SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - - pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex); - - // todo add windowRes iterator - int64_t lastTimestamp = -1; - int64_t startt = taosGetTimestampMs(); - - while (1) { - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:%p it is already killed, abort", pQInfo); - - tfree(pTableList); - tfree(posList); - tfree(pTree); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - int32_t pos = pTree->pNode[0].index; - - SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - - char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); - TSKEY ts = GET_INT64_VAL(b); - - assert(ts == pWindowRes->win.skey); - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); - if (num <= 0) { - cs.position[pos] += 1; - - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; - - // all input sources are exhausted - if (--numOfTables == 0) { - break; - } - } - } else { - if (ts == lastTimestamp) { // merge with the last one - doMerge(pRuntimeEnv, ts, pWindowRes, true); - } else { // copy data to disk buffer - if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - return -1; - } - - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - } - - doMerge(pRuntimeEnv, ts, pWindowRes, false); - buffer[0]->num += 1; - } - - lastTimestamp = ts; - - // move to the next element of current entry - int32_t currentPageId = pWindowRes->pageId; - - cs.position[pos] += 1; - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; - - // all input sources are exhausted - if (--numOfTables == 0) { - break; - } - } else { - // current page is not needed anymore - SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - if (pNextWindowRes->pageId != currentPageId) { - releaseResBufPage(pRuntimeEnv->pResultBuf, page); - } - } - } - - tLoserTreeAdjust(pTree, pos + pTree->numOfEntries); - } - - if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - - tfree(pTree); - tfree(pTableList); - tfree(posList); - return -1; - } - } - - int64_t endt = taosGetTimestampMs(); - -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif - - qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - - tfree(pTableList); - tfree(posList); - tfree(pTree); - -// tfree(pResultInfo); -// tfree(buf); - - return pQInfo->groupResInfo.numOfDataPages; -} - -int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. - int32_t pageId = -1; - int32_t capacity = pResultBuf->numOfRowsPerPage; - - int32_t remain = (int32_t) pQuery->sdata[0]->num; - int32_t offset = 0; - - while (remain > 0) { - int32_t rows = (remain > capacity)? capacity:remain; - assert(rows > 0); - - // get the output buffer page - tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); - buf->num = rows; - - // pagewisely copy to dest buffer - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - - char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; - char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; - memcpy(output, src, (size_t)(buf->num * bytes)); - } - - offset += rows; - remain -= rows; - - pGroupResInfo->numOfDataPages += 1; - } - - return TSDB_CODE_SUCCESS; -} - -void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; - pCtx[k].size = 1; - pCtx[k].startOffset = 0; - pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k); - - pQuery->sdata[k]->num = 0; - } -} - -static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { - if (pTableQueryInfo == NULL) { - return; - } - - // order has changed already - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { - // do nothing, no results - } else {// NOTE: even win.skey != lastKey, the results may not generated. - pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; - } - - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - - SWITCH_ORDER(pTableQueryInfo->cur.order); - pTableQueryInfo->cur.vgroupIndex = -1; - - // set the index at the end of time window - pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1; -} - -static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pWindowResInfo, int32_t order) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - bool closed = getResultRowStatus(pWindowResInfo, i); - if (!closed) { - continue; - } - - SResultRow *pRow = getResultRow(pWindowResInfo, i); - - // open/close the specified query for each group result - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functId = pQuery->pExpr1[j].base.functionId; - SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j); - - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - pInfo->complete = false; - } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - pInfo->complete = true; - } - } - } -} - -void disableFuncInReverseScan(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t order = pQuery->order.order; - - // group by normal columns and interval query on normal table - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { - disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); - } else { // for simple result of table query, - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor - int32_t functId = pQuery->pExpr1[j].base.functionId; - - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; - if (pCtx->resultInfo == NULL) { - continue; // resultInfo is NULL, means no data checked in previous scan - } - - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - pCtx->resultInfo->complete = false; - } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - pCtx->resultInfo->complete = true; - } - } - } -} - -static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); - SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); - - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - - // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide - // the start check timestamp of tsdbQueryHandle - STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); - pTableKeyInfo->lastKey = pCheckInfo->lastKey; - - assert(pCheckInfo->pTable == pTableKeyInfo->pTable); - } - } -} - -void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SWITCH_ORDER(pRuntimeEnv->pCtx[i].order); - } -} - -int32_t initResultRow(SResultRow *pResultRow) { - pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); - pResultRow->pageId = -1; - pResultRow->rowId = -1; - return TSDB_CODE_SUCCESS; -} - -void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - int32_t tid = 0; - int64_t uid = getResultInfoUId(pRuntimeEnv); - SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->aOutputBuf = pQuery->sdata[i]->data; - - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. - */ - SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i); - RESET_RESULT_INFO(pCellInfo); - pCtx->resultInfo = pCellInfo; - - // set the timestamp output buffer for top/bottom/diff query - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; - } - - memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pExpr1[i].bytes * pQuery->rec.capacity)); - } - - initCtxOutputBuf(pRuntimeEnv); -} - -void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - // reset the execution contexts - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; - assert(functionId != TSDB_FUNC_DIFF); - - // set next output position - if (IS_OUTER_FORWARD(aAggs[functionId].nStatus)) { - pRuntimeEnv->pCtx[j].aOutputBuf += pRuntimeEnv->pCtx[j].outputBytes * output; - } - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - /* - * NOTE: for top/bottom query, the value of first column of output (timestamp) are assigned - * in the procedure of top/bottom routine - * the output buffer in top/bottom routine is ptsOutputBuf, so we need to forward the output buffer - * - * diff function is handled in multi-output function - */ - pRuntimeEnv->pCtx[j].ptsOutputBuf = (char*)pRuntimeEnv->pCtx[j].ptsOutputBuf + TSDB_KEYSIZE * output; - } - - RESET_RESULT_INFO(pRuntimeEnv->pCtx[j].resultInfo); - } -} - -void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; +void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pExpr1[j].base.functionId; @@ -3803,10 +3342,10 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes)); - pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; + pRuntimeEnv->pCtx[i].pOutput = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; } } @@ -3824,22 +3363,18 @@ void setQueryStatus(SQuery *pQuery, int8_t status) { } } -bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { +bool needRepeatScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; bool toContinue = false; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pResult = getResultRow(pWindowResInfo, i); - if (!pResult->closed) { - continue; - } setResultOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pExpr1[j].base.functionId; if (functId == TSDB_FUNC_TS) { @@ -3878,15 +3413,11 @@ static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY SQueryStatusInfo info = { .status = pQuery->status, - .windowIndex = pRuntimeEnv->windowResInfo.curIndex, + .windowIndex = pRuntimeEnv->resultRowInfo.curIndex, .lastKey = start, }; TIME_WINDOW_COPY(info.w, pQuery->window); - TIME_WINDOW_COPY(info.curWindow, pTableQueryInfo->win); - - info.curWindow.skey = start; - return info; } @@ -3894,17 +3425,15 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; - pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor - if (pRuntimeEnv->pTSBuf) { - SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); - bool ret = tsBufNextPos(pRuntimeEnv->pTSBuf); + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); // save the cursor + if (pRuntimeEnv->pTsBuf) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); assert(ret); } // reverse order time range - pQuery->window = pStatus->curWindow; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -3939,9 +3468,9 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus SWITCH_ORDER(pQuery->order.order); switchCtxOrder(pRuntimeEnv); - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); - if (pRuntimeEnv->pTSBuf) { - pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pStatus->cur); + if (pRuntimeEnv->pTsBuf) { + pRuntimeEnv->pTsBuf->cur.order = pQuery->order.order; } SET_MASTER_SCAN_FLAG(pRuntimeEnv); @@ -3961,6 +3490,67 @@ static void restoreTimeWindow(STableGroupInfo* pTableGroupInfo, STsdbQueryCond* pKeyInfo->lastKey = pCond->twindow.skey; } +static void handleInterpolationQuery(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + + SQuery *pQuery = pRuntimeEnv->pQuery; + if (pQuery->numOfCheckedBlocks > 0 || !isPointInterpoQuery(pQuery)) { + return; + } + + SArray *prev = tsdbGetExternalRow(pRuntimeEnv->pQueryHandle, &pQInfo->memRef, TSDB_PREV_ROW); + SArray *next = tsdbGetExternalRow(pRuntimeEnv->pQueryHandle, &pQInfo->memRef, TSDB_NEXT_ROW); + if (prev == NULL || next == NULL) { + return; + } + + // setup the pCtx->start/end info and calculate the interpolation value + SColumnInfoData *startTs = taosArrayGet(prev, 0); + SColumnInfoData *endTs = taosArrayGet(next, 0); + + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + int32_t functionId = pQuery->pExpr1[i].base.functionId; + SColIndex *pColIndex = &pQuery->pExpr1[i].base.colInfo; + + if (!TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { + aAggs[functionId].xFunction(pCtx); + continue; + } + + SColumnInfoData *p = taosArrayGet(prev, pColIndex->colIndex); + SColumnInfoData *n = taosArrayGet(next, pColIndex->colIndex); + + assert(p->info.colId == pColIndex->colId); + + pCtx->start.key = *(TSKEY *)startTs->pData; + pCtx->end.key = *(TSKEY *)endTs->pData; + + if (p->info.type != TSDB_DATA_TYPE_BINARY && p->info.type != TSDB_DATA_TYPE_NCHAR) { + GET_TYPED_DATA(pCtx->start.val, double, p->info.type, p->pData); + GET_TYPED_DATA(pCtx->end.val, double, n->info.type, n->pData); + } else { // string pointer + pCtx->start.ptr = p->pData; + pCtx->end.ptr = n->pData; + } + + pCtx->param[2].i64 = (int8_t)pQuery->fillType; + pCtx->startTs = pQuery->window.skey; + if (pQuery->fillVal != NULL) { + if (isNull((const char *)&pQuery->fillVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value + if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->fillVal[i], pCtx->inputBytes, pCtx->inputType); + } + } + } + + aAggs[functionId].xFunction(pCtx); + } +} + void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; @@ -3970,9 +3560,11 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { // store the start query position SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv, start); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + if (!pRuntimeEnv->groupbyColumn && pRuntimeEnv->hasTagResults) { + setTagVal(pRuntimeEnv, pTableQueryInfo->pTable); + } while (1) { doScanAllDataBlocks(pRuntimeEnv); @@ -3981,16 +3573,12 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan - if (qstatus.lastKey != pTableQueryInfo->lastKey) { - qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; - } else { // the lastkey does not increase, which means no data checked yet + if (qstatus.lastKey == pTableQueryInfo->lastKey) { qDebug("QInfo:%p no results generated in this scan", pQInfo); } - - qstatus.lastKey = pTableQueryInfo->lastKey; } - if (!needScanDataBlocksAgain(pRuntimeEnv)) { + if (!needRepeatScan(pRuntimeEnv)) { // restore the status code and jump out of loop if (pRuntimeEnv->scanFlag == REPEAT_SCAN) { pQuery->status = qstatus.status; @@ -4003,52 +3591,52 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &qstatus.curWindow); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; + pRuntimeEnv->resultRowInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; + if (pRuntimeEnv->pTsBuf) { + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, cond.twindow.skey, cond.twindow.ekey); - - // check if query is killed or not - if (IS_QUERY_KILLED(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } } - if (!needReverseScan(pQuery)) { - return; - } + if (needReverseScan(pQuery)) { + setEnvBeforeReverseScan(pRuntimeEnv, &qstatus); - setEnvBeforeReverseScan(pRuntimeEnv, &qstatus); + // reverse scan from current position + qDebug("QInfo:%p start to reverse scan", pQInfo); + doScanAllDataBlocks(pRuntimeEnv); - // reverse scan from current position - qDebug("QInfo:%p start to reverse scan", pQInfo); - doScanAllDataBlocks(pRuntimeEnv); + clearEnvAfterReverseScan(pRuntimeEnv, &qstatus); + } - clearEnvAfterReverseScan(pRuntimeEnv, &qstatus); + handleInterpolationQuery(pQInfo); } void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol) { - closeAllTimeWindow(pWindowResInfo); + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; + if (pRuntimeEnv->groupbyColumn) { + closeAllResultRows(pWindowResInfo); } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *buf = pWindowResInfo->pResult[i]; - if (!isWindowResClosed(pWindowResInfo, i)) { + if (!isResultRowClosed(pWindowResInfo, i)) { continue; } @@ -4084,9 +3672,7 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void* pTable, STimeWindow win, void* buf) { - SQuery *pQuery = pRuntimeEnv->pQuery; - +static STableQueryInfo *createTableQueryInfo(SQuery* pQuery, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { STableQueryInfo *pTableQueryInfo = buf; pTableQueryInfo->win = win; @@ -4096,9 +3682,9 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void pTableQueryInfo->cur.vgroupIndex = -1; // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { + if (QUERY_IS_INTERVAL_QUERY(pQuery) || groupbyColumn) { int32_t initialSize = 128; - int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4114,7 +3700,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { } tVariantDestroy(&pTableQueryInfo->tag); - cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); + cleanupResultRowInfo(&pTableQueryInfo->resInfo); } /** @@ -4123,27 +3709,20 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { * @param pDataBlockInfo */ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; - - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { - setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); - } - if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { return; } - uint64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); - if (pResultRow == NULL) { - return; - } + assert (pResultRow != NULL); /* * not assign result buffer yet, add new result buffer @@ -4170,11 +3749,11 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, page); + pCtx->pOutput = getPosInResultPage(pRuntimeEnv, i, pResult, page); int32_t functionId = pQuery->pExpr1[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; } /* @@ -4199,60 +3778,96 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe continue; } - pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage); + pCtx->pOutput = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage); pCtx->currentStage = 0; int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; } - + if (!pCtx->resultInfo->initialized) { aAggs[functionId].init(pCtx); } } } -int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQueryInfo) { +int32_t setTimestampListJoinInfo(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - setTagVal(pRuntimeEnv, pTable, pQInfo->tsdb); + assert(pRuntimeEnv->pTsBuf != NULL); // both the master and supplement scan needs to set the correct ts comp start position - if (pRuntimeEnv->pTSBuf != NULL) { - tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; - - if (pTableQueryInfo->cur.vgroupIndex == -1) { - tVariantAssign(&pTableQueryInfo->tag, pTag); + tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag); + if (pTableQueryInfo->cur.vgroupIndex == -1) { + tVariantAssign(&pTableQueryInfo->tag, pTag); - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); - } else { - qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64Key); - } - - return false; - } + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, &pTableQueryInfo->tag); - // keep the cursor info of current meter - pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + // failed to find data with the specified tag value and vnodeId + if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64); } + return false; + } + + // keep the cursor info of current meter + pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); + + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } + } + + return 0; +} + +int32_t setParamValue(SQueryRuntimeEnv* pRuntimeEnv) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + if (pRuntimeEnv->prevResult == NULL || pRuntimeEnv->groupbyColumn) { + return TSDB_CODE_SUCCESS; + } + + int32_t numOfExprs = pQuery->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo = &(pQuery->pExpr1[i]); + if(pExprInfo->base.functionId != TSDB_FUNC_STDDEV_DST) { + continue; + } + + SSqlFuncMsg* pFuncMsg = &pExprInfo->base; + + pRuntimeEnv->pCtx[i].param[0].arr = NULL; + pRuntimeEnv->pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + int32_t numOfGroup = (int32_t) taosArrayGetSize(pRuntimeEnv->prevResult); + for(int32_t j = 0; j < numOfGroup; ++j) { + SInterResult *p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (pQuery->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQuery->tagLen) == 0) { + + int32_t numOfCols = (int32_t) taosArrayGetSize(p->pResult); + for(int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pFuncMsg->colInfo.colId) { + pRuntimeEnv->pCtx[i].param[0].arr = pres->pResult; + break; + } + } } } } @@ -4289,16 +3904,15 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { /** * In handling the both ascending and descending order super table query, we need to find the first qualified * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional * operations involve. */ STimeWindow w = TSWINDOW_INITIALIZER; - SResultRowInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo *pWindowResInfo = &pTableQueryInfo->resInfo; TSKEY sk = MIN(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); - pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { if (!QUERY_IS_ASC_QUERY(pQuery)) { @@ -4316,7 +3930,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { bool requireTimestamp(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; i++) { int32_t functionId = pQuery->pExpr1[i].base.functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { + if ((aAggs[functionId].status & TSDB_FUNCSTATE_NEED_TS) != 0) { return true; } } @@ -4337,92 +3951,81 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static int32_t doCopyToSData(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType) { + void* qinfo = GET_QINFO_ADDR(pRuntimeEnv); + SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t numOfResult = 0; - int32_t startIdx = 0; - int32_t step = -1; + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + int32_t numOfResult = (int32_t) pQuery->rec.rows; // there are already exists result rows - qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); - int32_t totalSet = numOfClosedTimeWindow(pResultInfo); - SResultRow** result = pResultInfo->pResult; + int32_t start = 0; + int32_t step = -1; + qDebug("QInfo:%p start to copy data from windowResInfo to output buf", qinfo); if (orderType == TSDB_ORDER_ASC) { - startIdx = pQInfo->groupIndex; + start = pGroupResInfo->index; step = 1; } else { // desc order copy all data - startIdx = totalSet - pQInfo->groupIndex - 1; + start = numOfRows - pGroupResInfo->index - 1; step = -1; } - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { - if (result[i]->numOfRows == 0) { - pQInfo->groupIndex += 1; - pGroupResInfo->rowId = 0; + for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { + SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i); + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; continue; } - int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId; - int32_t oldOffset = pGroupResInfo->rowId; + int32_t numOfRowsToCopy = pRow->numOfRows; - /* - * current output space is not enough to accommodate all data of this page, only partial results - * will be copied to SQuery object's result buffer - */ - if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { - numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; - pGroupResInfo->rowId += numOfRowsToCopy; - } else { - pGroupResInfo->rowId = 0; - pQInfo->groupIndex += 1; + //current output space is not enough to accommodate all data of this page, prepare more space + if (numOfRowsToCopy > (pQuery->rec.capacity - numOfResult)) { + int32_t newSize = (int32_t) (pQuery->rec.capacity + (numOfRowsToCopy - numOfResult)); + expandBuffer(pRuntimeEnv, newSize, GET_QINFO_ADDR(pRuntimeEnv)); } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId); + pGroupResInfo->index += 1; + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page); - memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); + char *in = getPosInResultPage(pRuntimeEnv, j, pRow, page); + memcpy(out, in, size * numOfRowsToCopy); } numOfResult += numOfRowsToCopy; - if (numOfResult == pQuery->rec.capacity) { + if (numOfResult == pQuery->rec.capacity) { // output buffer is full break; } } - qDebug("QInfo:%p copy data to query buf completed", pQInfo); - -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, numOfResult); -#endif + qDebug("QInfo:%p copy data to query buf completed", qinfo); return numOfResult; } /** - * copyFromWindowResToSData support copy data in ascending/descending order + * copyToOutputBuf support copy data in ascending/descending order * For interval query of both super table and table, copy the data in ascending order, since the output results are * ordered in SWindowResutl already. While handling the group by query for both table and super table, * all group result are completed already. * * @param pQInfo - * @param result - */ -void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo, orderType); + * @param result + */ +void copyToOutputBuf(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + SGroupResInfo *pGroupResInfo = &pQInfo->groupResInfo; - pQuery->rec.rows += numOfResult; + assert(pQuery->rec.rows == 0 && pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + if (!hasRemainData(pGroupResInfo)) { + return; + } - assert(pQuery->rec.rows <= pQuery->rec.capacity); + int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; + pQuery->rec.rows = doCopyToSData(&pQInfo->runtimeEnv, pGroupResInfo, orderType); } static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { @@ -4433,8 +4036,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { return; } - for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - SResultRow *pResult = pRuntimeEnv->windowResInfo.pResult[i]; + for (int32_t i = 0; i < pRuntimeEnv->resultRowInfo.size; ++i) { + SResultRow *pResult = pRuntimeEnv->resultRowInfo.pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pRuntimeEnv->pCtx[j].functionId; @@ -4453,28 +4056,35 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo * pResultRowInfo = &pTableQueryInfo->resInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); + } + + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); } } -bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { +bool hasNotReturnedResults(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo *pFillInfo = pRuntimeEnv->pFillInfo; + if (!Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + return false; + } + if (pQuery->limit.limit > 0 && pQuery->rec.total >= pQuery->limit.limit) { return false; } if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { // There are results not returned to client yet, so filling applied to the remain result is required firstly. - int32_t remain = taosNumOfRemainRows(pFillInfo); - if (remain > 0) { + if (taosFillHasMoreResults(pFillInfo)) { return true; } @@ -4487,16 +4097,11 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { * set is the FIRST result block, the gap between the start time of query time window and the timestamp of the * first result row in the actual result set will fill nothing. */ - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - int32_t numOfTotal = (int32_t)getNumOfResWithFill(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity); - return numOfTotal > 0; - } - - } else { - // there are results waiting for returned to client. - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && - (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) && - (pRuntimeEnv->windowResInfo.size > 0)) { + int32_t numOfTotal = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity); + return numOfTotal > 0; + } else { // there are results waiting for returned to client. + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && hasRemainData(pGroupResInfo) && + (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery))) { return true; } } @@ -4531,7 +4136,9 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); + int32_t total = 0; STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL); + while(item) { STableIdInfo* pDst = (STableIdInfo*)data; pDst->uid = htobe64(item->uid); @@ -4539,9 +4146,14 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data pDst->key = htobe64(item->key); data += sizeof(STableIdInfo); + total++; + + qDebug("QInfo:%p set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo, item->tid, item->uid, item->key); item = taosHashIterate(pQInfo->arrTableIdInfo, item); } + qDebug("QInfo:%p set %d subscribe info", pQInfo, total); + // Check if query is completed or not for stable query or normal table query respectively. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (pQInfo->runtimeEnv.stableQuery) { @@ -4549,15 +4161,15 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data setQueryStatus(pQuery, QUERY_OVER); } } else { - if (!queryHasRemainResForTableQuery(&pQInfo->runtimeEnv)) { + if (!hasNotReturnedResults(&pQInfo->runtimeEnv, &pQInfo->groupResInfo)) { setQueryStatus(pQuery, QUERY_OVER); } } } } -int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t *numOfFilled) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); +int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst) { + SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; @@ -4576,8 +4188,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); ret -= (int32_t)pQuery->limit.offset; - // todo !!!!there exactly number of interpo is not valid. - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { //???pExpr1 or pExpr2 memmove(pDst[i]->data, pDst[i]->data + pQuery->pExpr1[i].bytes * pQuery->limit.offset, ret * pQuery->pExpr1[i].bytes); } @@ -4590,17 +4201,18 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int pQuery->limit.offset - ret); pQuery->limit.offset -= ret; - pQuery->rec.rows = 0; ret = 0; } - if (!queryHasRemainResForTableQuery(pRuntimeEnv)) { - return ret; + // no data in current data after fill + int32_t numOfTotal = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, (int32_t)pQuery->rec.capacity); + if (numOfTotal == 0) { + return 0; } } } -static void queryCostStatis(SQInfo *pQInfo) { +void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; @@ -4612,8 +4224,13 @@ static void queryCostStatis(SQInfo *pQInfo) { pSummary->elapsedTime += pSummary->firstStageMergeTime; SResultRowPool* p = pQInfo->runtimeEnv.pool; - pSummary->winInfoSize = getResultRowPoolMemSize(p); - pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + if (p != NULL) { + pSummary->winInfoSize = getResultRowPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + } else { + pSummary->winInfoSize = 0; + pSummary->numOfTimeWindows = 0; + } qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, @@ -4659,7 +4276,59 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBloc qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); } - +static void freeTableBlockDist(STableBlockDist *pTableBlockDist) { + if (pTableBlockDist != NULL) { + taosArrayDestroy(pTableBlockDist->dataBlockInfos); + free(pTableBlockDist->result); + free(pTableBlockDist); + } +} +static int32_t getPercentileFromSortedArray(const SArray* pArray, double rate) { + int32_t len = (int32_t)taosArrayGetSize(pArray); + if (len <= 0) { + return 0; + } + assert(rate >= 0 && rate <= 1.0); + int idx = (int32_t)((len - 1) * rate); + return ((SDataBlockInfo *)(taosArrayGet(pArray, idx)))->rows; +} +static int compareBlockInfo(const void *pLeft, const void *pRight) { + int32_t left = ((SDataBlockInfo *)pLeft)->rows; + int32_t right = ((SDataBlockInfo *)pRight)->rows; + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +static void generateBlockDistResult(STableBlockDist *pTableBlockDist) { + if (pTableBlockDist == NULL) { + return; + } + int64_t min = 0, max = 0, avg = 0; + SArray* blockInfos= pTableBlockDist->dataBlockInfos; + int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); + for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) { + SDataBlockInfo *blockInfo = taosArrayGet(blockInfos, i); + int64_t rows = blockInfo->rows; + min = MIN(min, rows); + max = MAX(max, rows); + totalRows += rows; + } + avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; + + taosArraySort(blockInfos, compareBlockInfo); + + int sz = sprintf(pTableBlockDist->result, + "summery: \n\t 5th=[%d], 25th=[%d], 50th=[%d],75th=[%d], 95th=[%d], 99th=[%d] \n\t min=[%"PRId64"], max=[%"PRId64"], avg = [%"PRId64"] \n\t totalRows=[%"PRId64"], totalBlocks=[%"PRId64"] \n\t seekHeaderTimeCost=[%"PRId64"(us)] \n\t rowsInMem=[%"PRId64"]", + getPercentileFromSortedArray(blockInfos, 0.05), getPercentileFromSortedArray(blockInfos, 0.25), getPercentileFromSortedArray(blockInfos, 0.50), + getPercentileFromSortedArray(blockInfos, 0.75), getPercentileFromSortedArray(blockInfos, 0.95), getPercentileFromSortedArray(blockInfos, 0.99), + min, max, avg, + totalRows, totalBlocks, + pTableBlockDist->firstSeekTimeUs, + pTableBlockDist->numOfRowsInMemTable); + UNUSED(sz); + return; +} void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; @@ -4675,7 +4344,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; while (tsdbNextDataBlock(pQueryHandle)) { - if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -4701,7 +4370,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; assert(pQuery->limit.offset == 0); STimeWindow tw = *win; @@ -4729,10 +4398,10 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w TSKEY key = pTableQueryInfo->win.skey; pWindowResInfo->prevSKey = tw.skey; - int32_t index = pRuntimeEnv->windowResInfo.curIndex; + int32_t index = pRuntimeEnv->resultRowInfo.curIndex; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); - pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index + pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, @@ -4740,8 +4409,9 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w return key; } else { // do nothing - pQuery->window.skey = tw.skey; + pQuery->window.skey = tw.skey; pWindowResInfo->prevSKey = tw.skey; + pTableQueryInfo->lastKey = tw.skey; return tw.skey; } @@ -4751,26 +4421,14 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { SQuery *pQuery = pRuntimeEnv->pQuery; - - // get the first unclosed time window - bool assign = false; - for(int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - if (pRuntimeEnv->windowResInfo.pResult[i]->closed) { - continue; - } - - assign = true; - *start = pRuntimeEnv->windowResInfo.pResult[i]->win.skey; - } - - if (!assign) { - *start = pQuery->current->lastKey; + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert(*start <= pQuery->current->lastKey); + } else { + assert(*start >= pQuery->current->lastKey); } - assert(*start <= pQuery->current->lastKey); - // if queried with value filter, do NOT forward query start position - if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { return true; } @@ -4779,11 +4437,12 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { * pQuery->limit.offset times. Since hole exists, pQuery->interval.interval*pQuery->limit.offset value is * not valid. otherwise, we only forward pQuery->limit.offset number of points */ - assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); + assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; @@ -4793,13 +4452,10 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { if (QUERY_IS_ASC_QUERY(pQuery)) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } } else { getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } @@ -4809,19 +4465,25 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { while (pQuery->limit.offset > 0) { STimeWindow tw = win; - if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; pWindowResInfo->prevSKey = win.skey; + + // current time window is aligned with blockInfo.window.ekey + // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; + if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { + pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + } } - // current window does not ended in current data block, try next data block - getNextTimeWindow(pQuery, &tw); if (pQuery->limit.offset == 0) { *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); return true; } + // current window does not ended in current data block, try next data block + getNextTimeWindow(pQuery, &tw); + /* * If the next time window still starts from current data block, * load the primary timestamp column first, and then find the start position for the next queried time window. @@ -4829,13 +4491,12 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required * time window resides in current data block. */ - if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { + + SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; } @@ -4888,7 +4549,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) - && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) + && (!isGroupbyColumn(pQuery->pGroupbyExpr)) && (!isFixedOutputQuery(pRuntimeEnv)) ) { SArray* pa = GET_TABLEGROUP(pQInfo, 0); @@ -4902,7 +4563,6 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) // update the query time window pQuery->window = cond.twindow; - if (pQInfo->tableGroupInfo.numOfTables == 0) { pQInfo->tableqinfoGroupInfo.numOfTables = 0; } else { @@ -4955,7 +4615,7 @@ static SFillColInfo* createFillColInfo(SQuery* pQuery) { return pFillCol; } -int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, SArray* prevResult, void *tsdb, int32_t vgId, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4963,6 +4623,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery); pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); pRuntimeEnv->timeWindowInterpo = timeWindowInterpoRequired(pQuery); + pRuntimeEnv->prevResult = prevResult; setScanLimitationByResultBuffer(pQuery); @@ -4973,17 +4634,19 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; + pQInfo->groupResInfo.totalGroup = (int32_t) (isSTableQuery? GET_NUM_OF_TABLEGROUP(pQInfo):0); pRuntimeEnv->pQuery = pQuery; - pRuntimeEnv->pTSBuf = pTsBuf; + pRuntimeEnv->pTsBuf = pTsBuf; pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->groupbyNormalCol = isGroupbyNormalCol(pQuery->pGroupbyExpr); + pRuntimeEnv->groupbyColumn = isGroupbyColumn(pQuery->pGroupbyExpr); + pRuntimeEnv->stabledev = isStabledev(pQuery); if (pTsBuf != NULL) { - int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + int16_t order = (pQuery->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); } int32_t ps = DEFAULT_PAGE_SIZE; @@ -4999,18 +4662,18 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; + if (pRuntimeEnv->groupbyColumn) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_INT; // group id } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, type); + code = initResultRowInfo(&pRuntimeEnv->resultRowInfo, 8, type); if (code != TSDB_CODE_SUCCESS) { return code; } } - } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { + } else if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { int32_t numOfResultRows = getInitialPageNum(pQInfo); getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); @@ -5019,20 +4682,20 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->groupbyColumn) { type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, type); + code = initResultRowInfo(&pRuntimeEnv->resultRowInfo, numOfResultRows, type); if (code != TSDB_CODE_SUCCESS) { return code; } } // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, pQuery->order.order); + code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQInfo->tableGroupInfo.numOfTables, pQuery->order.order, pQInfo->vgId); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5046,7 +4709,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w); int32_t numOfCols = getNumOfFinalResCol(pQuery); - pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, numOfCols, + pRuntimeEnv->pFillInfo = taosCreateFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, numOfCols, pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision, pQuery->fillType, pColInfo, pQInfo); } @@ -5071,15 +4734,22 @@ static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTa SQuery* pQuery = pQInfo->runtimeEnv.pQuery; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { - setExecutionContext(pQInfo, pTableQueryInfo->groupIndex, pBlockInfo->window.ekey + step); - } else { // interval query - TSKEY nextKey = pBlockInfo->window.skey; - setIntervalQueryRange(pQInfo, nextKey); + if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { + setTagVal(pRuntimeEnv, pTableQueryInfo->pTable); + } + + if (pRuntimeEnv->pTsBuf != NULL) { + setTimestampListJoinInfo(pQInfo, pTableQueryInfo); + } - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { - setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (!QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery)) { + setIntervalQueryRange(pQInfo, pBlockInfo->window.ekey); + } else { + setIntervalQueryRange(pQInfo, pBlockInfo->window.skey); } + } else { // non-interval query + setExecutionContext(pQInfo, pTableQueryInfo->groupIndex, pBlockInfo->window.ekey + step); } } @@ -5100,7 +4770,7 @@ static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTa static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; - SQueryCostInfo* summary = &pRuntimeEnv->summary; + SQueryCostInfo* summary = &pRuntimeEnv->summary; int64_t st = taosGetTimestampMs(); @@ -5112,7 +4782,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; - if (IS_QUERY_KILLED(pQInfo)) { + if (isQueryKilled(pQInfo)) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5125,15 +4795,24 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { pQuery->current = *pTableQueryInfo; doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); } + if (pRuntimeEnv->stabledev) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + if (pQuery->pExpr1[i].base.functionId == TSDB_FUNC_STDDEV_DST) { + setParamValue(pRuntimeEnv); + break; + } + } + } + uint32_t status = 0; SDataStatis *pStatis = NULL; SArray *pDataBlock = NULL; - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->resInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); if (ret != TSDB_CODE_SUCCESS) { break; } @@ -5170,8 +4849,8 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { SArray *group = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(group, index); - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { - setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb); + if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { + setTagVal(pRuntimeEnv, pCheckInfo->pTable); } STableId* id = TSDB_TABLEID(pCheckInfo->pTable); @@ -5183,6 +4862,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { .order = pQuery->order.order, .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, + .loadExternalRows = false, }; // todo refactor @@ -5208,61 +4888,61 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { longjmp(pRuntimeEnv->env, terrno); } - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64); } return false; } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, cur.blockIndex, cur.tsIndex); } } } else { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); if (tVariantCompare(elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { - STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem1)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64); } return false; } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, cur.blockIndex, cur.tsIndex); } } } else { - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pRuntimeEnv->cur); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); } else { - qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, cur.blockIndex, cur.tsIndex); } } } @@ -5277,6 +4957,7 @@ STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) { .colList = pQuery->colList, .order = pQuery->order.order, .numOfCols = pQuery->numOfCols, + .loadExternalRows = false, }; TIME_WINDOW_COPY(cond.twindow, *win); @@ -5333,7 +5014,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayClone(group); + SArray *tx = taosArrayDup(group); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; @@ -5357,7 +5038,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); - setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); + setTagVal(pRuntimeEnv, taosArrayGetP(s, 0)); taosArrayDestroy(s); // here we simply set the first table as current table @@ -5383,7 +5064,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } } - } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query + } else if (pRuntimeEnv->groupbyColumn) { // group-by on normal columns query while (pQInfo->groupIndex < numOfGroups) { SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); @@ -5393,7 +5074,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayClone(group); + SArray *tx = taosArrayDup(group); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; @@ -5416,16 +5097,16 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); - setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); + setTagVal(pRuntimeEnv, taosArrayGetP(s, 0)); // here we simply set the first table as current table scanMultiTableDataBlocks(pQInfo); pQInfo->groupIndex += 1; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + taosArrayDestroy(s); // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; if (pWindowResInfo->size <= 0) { continue; } @@ -5442,21 +5123,21 @@ static void sequentialTableProcess(SQInfo *pQInfo) { qDebug("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size, pQInfo->groupIndex); - int32_t currentGroupIndex = pQInfo->groupIndex; pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; - - ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size); - copyFromWindowResToSData(pQInfo, pWindowResInfo); + if (pWindowResInfo->size > pQuery->rec.capacity) { + expandBuffer(pRuntimeEnv, pWindowResInfo->size, pQInfo); + } - pQInfo->groupIndex = currentGroupIndex; // restore the group index + initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, 0); + copyToOutputBuf(pQInfo, pWindowResInfo); assert(pQuery->rec.rows == pWindowResInfo->size); - clearClosedTimeWindow(pRuntimeEnv); + resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->resultRowInfo); + cleanupGroupResInfo(&pQInfo->groupResInfo); break; } - } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL && !isTSCompQuery(pQuery)) { + } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTsBuf == NULL && !isTsCompQuery(pQuery)) { //super table projection query with identical query time range for all tables. SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; resetDefaultResInfoOutputBuf(pRuntimeEnv); @@ -5479,19 +5160,27 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // return; // } + if (pQuery->prjInfo.vgroupLimit != -1) { + assert(pQuery->limit.limit == -1 && pQuery->limit.offset == 0); + } else if (pQuery->limit.limit != -1) { + assert(pQuery->prjInfo.vgroupLimit == -1); + } + + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + bool hasMoreBlock = true; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); SQueryCostInfo *summary = &pRuntimeEnv->summary; while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) { summary->totalBlocks += 1; - if (IS_QUERY_KILLED(pQInfo)) { + if (isQueryKilled(pQInfo)) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); STableQueryInfo **pTableQueryInfo = - (STableQueryInfo **)taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); + (STableQueryInfo **) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); if (pTableQueryInfo == NULL) { break; } @@ -5500,14 +5189,33 @@ static void sequentialTableProcess(SQInfo *pQInfo) { doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); if (pRuntimeEnv->hasTagResults) { - setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb); + setTagVal(pRuntimeEnv, pQuery->current->pTable); + } + + if (pQuery->prjInfo.vgroupLimit > 0 && pQuery->current->resInfo.size > pQuery->prjInfo.vgroupLimit) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; + continue; + } + + // it is a super table ordered projection query, check for the number of output for each vgroup + if (pQuery->prjInfo.vgroupLimit > 0 && pQuery->rec.rows >= pQuery->prjInfo.vgroupLimit) { + if (QUERY_IS_ASC_QUERY(pQuery) && blockInfo.window.skey >= pQuery->prjInfo.ts) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; + continue; + } else if (!QUERY_IS_ASC_QUERY(pQuery) && blockInfo.window.ekey <= pQuery->prjInfo.ts) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; + continue; + } } uint32_t status = 0; SDataStatis *pStatis = NULL; SArray *pDataBlock = NULL; - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->resInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); if (ret != TSDB_CODE_SUCCESS) { break; @@ -5519,7 +5227,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { continue; } - ensureOutputBuffer(pRuntimeEnv, &blockInfo); + ensureOutputBuffer(pRuntimeEnv, blockInfo.rows); + int64_t prev = getNumOfResult(pRuntimeEnv); + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); @@ -5530,17 +5240,29 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->rec.rows = getNumOfResult(pRuntimeEnv); + int64_t inc = pQuery->rec.rows - prev; + pQuery->current->resInfo.size += (int32_t) inc; + // the flag may be set by tableApplyFunctionsOnBlock, clear it here CLEAR_QUERY_STATUS(pQuery, QUERY_COMPLETED); updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); - skipResults(pRuntimeEnv); - // the limitation of output result is reached, set the query completed - if (limitResults(pRuntimeEnv)) { - setQueryStatus(pQuery, QUERY_COMPLETED); - SET_STABLE_QUERY_OVER(pQInfo); - break; + if (pQuery->prjInfo.vgroupLimit >= 0) { + if (((pQuery->rec.rows + pQuery->rec.total) < pQuery->prjInfo.vgroupLimit) || ((pQuery->rec.rows + pQuery->rec.total) > pQuery->prjInfo.vgroupLimit && prev < pQuery->prjInfo.vgroupLimit)) { + if (QUERY_IS_ASC_QUERY(pQuery) && pQuery->prjInfo.ts < blockInfo.window.ekey) { + pQuery->prjInfo.ts = blockInfo.window.ekey; + } else if (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->prjInfo.ts > blockInfo.window.skey) { + pQuery->prjInfo.ts = blockInfo.window.skey; + } + } + } else { + // the limitation of output result is reached, set the query completed + skipResults(pRuntimeEnv); + if (limitOperator(pQuery, pQInfo)) { + SET_STABLE_QUERY_OVER(pQInfo); + break; + } } // while the output buffer is full or limit/offset is applied, query may be paused here @@ -5560,8 +5282,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * If the subgroup index is larger than 0, results generated by group by tbname,k is existed. * we need to return it to client in the first place. */ - if (pQInfo->groupIndex > 0) { - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + if (hasRemainData(&pQInfo->groupResInfo)) { + copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); pQuery->rec.total += pQuery->rec.rows; if (pQuery->rec.rows > 0) { @@ -5575,14 +5297,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } resetDefaultResInfoOutputBuf(pRuntimeEnv); - resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); + resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->resultRowInfo); SArray *group = GET_TABLEGROUP(pQInfo, 0); assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); while (pQInfo->tableIndex < pQInfo->tableqinfoGroupInfo.numOfTables) { - if (IS_QUERY_KILLED(pQInfo)) { + if (isQueryKilled(pQInfo)) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5604,7 +5326,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { skipResults(pRuntimeEnv); // the limitation of output result is reached, set the query completed - if (limitResults(pRuntimeEnv)) { + if (limitOperator(pQuery, pQInfo)) { SET_STABLE_QUERY_OVER(pQInfo); break; } @@ -5627,8 +5349,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } - if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + if (pRuntimeEnv->pTsBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; } } else { @@ -5659,12 +5381,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * * Only the ts-comp query requires the finalizer function to be executed here. */ - if (isTSCompQuery(pQuery)) { + if (isTsCompQuery(pQuery)) { finalizeQueryResult(pRuntimeEnv); } - if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + if (pRuntimeEnv->pTsBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; } qDebug("QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 @@ -5674,7 +5396,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } -static void doSaveContext(SQInfo *pQInfo) { +static int32_t doSaveContext(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -5682,8 +5404,8 @@ static void doSaveContext(SQInfo *pQInfo) { SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); - if (pRuntimeEnv->pTSBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); + if (pRuntimeEnv->pTsBuf != NULL) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); } STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); @@ -5700,9 +5422,7 @@ static void doSaveContext(SQInfo *pQInfo) { pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + return (pRuntimeEnv->pSecQueryHandle == NULL)? -1:0; } static void doRestoreContext(SQInfo *pQInfo) { @@ -5710,16 +5430,17 @@ static void doRestoreContext(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + SWITCH_ORDER(pQuery->order.order); - if (pRuntimeEnv->pTSBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); + if (pRuntimeEnv->pTsBuf != NULL) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); } switchCtxOrder(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { +static void doCloseAllTimeWindow(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (QUERY_IS_INTERVAL_QUERY(pQuery)) { @@ -5730,33 +5451,26 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { size_t num = taosArrayGetSize(group); for (int32_t j = 0; j < num; ++j) { STableQueryInfo* item = taosArrayGetP(group, j); - closeAllTimeWindow(&item->windowResInfo); + closeAllResultRows(&item->resInfo); } } } else { // close results for group result - closeAllTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + closeAllResultRows(&pQInfo->runtimeEnv.resultRowInfo); } } static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQInfo->groupIndex > 0) { - /* - * if the groupIndex > 0, the query process must be completed yet, we only need to - * copy the data into output buffer - */ + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif } else { - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); } - qDebug("QInfo:%p current:%"PRId64", total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); + qDebug("QInfo:%p current:%"PRId64", total:%"PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } @@ -5768,50 +5482,46 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qDebug("QInfo:%p master scan completed, elapsed time: %" PRId64 "ms, reverse scan start", pQInfo, el); // query error occurred or query is killed, abort current execution - if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } // close all time window results - doCloseAllTimeWindowAfterScan(pQInfo); + doCloseAllTimeWindow(pQInfo); if (needReverseScan(pQuery)) { - doSaveContext(pQInfo); - - el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); - - doRestoreContext(pQInfo); + int32_t code = doSaveContext(pQInfo); + if (code == TSDB_CODE_SUCCESS) { + el = scanMultiTableDataBlocks(pQInfo); + qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); + doRestoreContext(pQInfo); + } else { + pQInfo->code = code; + } } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); } setQueryStatus(pQuery, QUERY_COMPLETED); - if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { - if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { - copyResToQueryResultBuf(pQInfo, pQuery); - -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif - } + copyResToQueryResultBuf(pQInfo, pQuery); } else { // not a interval query - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, 0); + copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); } // handle the limitation of output buffer qDebug("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *) param; SExprInfo* pExprInfo = (SExprInfo*) pSupport->exprList; @@ -5837,7 +5547,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { tFilePage **data = calloc(pQuery->numOfExpr2, POINTER_BYTES); for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { int32_t bytes = pQuery->pExpr2[i].bytes; - data[i] = (tFilePage *)malloc(bytes * pQuery->rec.rows + sizeof(tFilePage)); + data[i] = (tFilePage *)malloc((size_t)(bytes * pQuery->rec.rows) + sizeof(tFilePage)); } arithSup.offset = 0; @@ -5859,19 +5569,19 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { if (pSqlFunc->functionId == pQuery->pExpr1[j].base.functionId && pSqlFunc->colInfo.colId == pQuery->pExpr1[j].base.colInfo.colId) { - memcpy(data[i]->data, pQuery->sdata[j]->data, pQuery->pExpr1[j].bytes * pQuery->rec.rows); + memcpy(data[i]->data, pQuery->sdata[j]->data, (size_t)(pQuery->pExpr1[j].bytes * pQuery->rec.rows)); break; } } } else { arithSup.pArithExpr = pExpr; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, getArithemicInputSrc); } } for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - memcpy(pQuery->sdata[i]->data, data[i]->data, pQuery->pExpr2[i].bytes * pQuery->rec.rows); + memcpy(pQuery->sdata[i]->data, data[i]->data, (size_t)(pQuery->pExpr2[i].bytes * pQuery->rec.rows)); } for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { @@ -5888,7 +5598,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; * select count(*) from table_name group by status_column; */ -static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableAggregationProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -5896,8 +5606,6 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) return; } - pQuery->current = pTableInfo; // set current query table info - scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); finalizeQueryResult(pRuntimeEnv); @@ -5905,22 +5613,21 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) pQuery->rec.rows = getNumOfResult(pRuntimeEnv); doSecondaryArithmeticProcess(pQuery); - if (IS_QUERY_KILLED(pQInfo)) { + if (isQueryKilled(pQInfo)) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } + // TODO limit/offset refactor to be one operator skipResults(pRuntimeEnv); - limitResults(pRuntimeEnv); + limitOperator(pQuery, pQInfo); } -static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableProjectionProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - pQuery->current = pTableInfo; - // for ts_comp query, re-initialized is not allowed - if (!isTSCompQuery(pQuery)) { + SQuery *pQuery = pRuntimeEnv->pQuery; + if (!isTsCompQuery(pQuery)) { resetDefaultResInfoOutputBuf(pRuntimeEnv); } @@ -5954,7 +5661,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) resetDefaultResInfoOutputBuf(pRuntimeEnv); } - limitResults(pRuntimeEnv); + limitOperator(pQuery, pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { qDebug("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->current->lastKey, pQuery->window.ekey); @@ -5963,33 +5670,40 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) taosHashPut(pQInfo->arrTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); } - if (!isTSCompQuery(pQuery)) { + if (!isTsCompQuery(pQuery)) { assert(pQuery->rec.rows <= pQuery->rec.capacity); } } -static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQuery *pQuery = pRuntimeEnv->pQuery; +static void copyAndFillResult(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - while (1) { - scanOneTableDataBlocks(pRuntimeEnv, start); + while(1) { + copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); + doSecondaryArithmeticProcess(pQuery); - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - finalizeQueryResult(pRuntimeEnv); + TSKEY lastKey = 0; + if (!hasRemainData(&pQInfo->groupResInfo)) { + lastKey = pQuery->window.ekey; + } else { + lastKey = ((TSKEY*)pQuery->sdata[0]->data)[pQuery->rec.rows - 1]; + } + + assert(lastKey <= pQuery->window.ekey); + + taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, lastKey); + taosFillSetDataBlockFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage **)pQuery->sdata); - // here we can ignore the records in case of no interpolation - // todo handle offset, in case of top/bottom interval query - if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 && - pQuery->fillType == TSDB_FILL_NONE) { - // maxOutput <= 0, means current query does not generate any results - int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo); + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata); - int32_t c = (int32_t)(MIN(numOfClosed, pQuery->limit.offset)); - clearFirstNWindowRes(pRuntimeEnv, c); - pQuery->limit.offset -= c; + if (pQuery->rec.rows > 0) { + limitOperator(pQuery, pQInfo); + break; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) { + // here the pQuery->rec.rows == 0 + if (!hasRemainData(&pQInfo->groupResInfo) && !taosFillHasMoreResults(pRuntimeEnv->pFillInfo)) { break; } } @@ -5998,15 +5712,12 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) // handle time interval query on table static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; - pQuery->current = pTableInfo; - int32_t numOfFilled = 0; - TSKEY newStartKey = TSKEY_INITIAL_VAL; + TSKEY newStartKey = QUERY_IS_ASC_QUERY(pQuery)? INT64_MIN:INT64_MAX; // skip blocks without load the actual data block from file if no filter condition present - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { setQueryStatus(pQuery, QUERY_COMPLETED); @@ -6014,104 +5725,68 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { } } - while (1) { - tableIntervalProcessImpl(pRuntimeEnv, newStartKey); - - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - pQInfo->groupIndex = 0; // always start from 0 - pQuery->rec.rows = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + scanOneTableDataBlocks(pRuntimeEnv, newStartKey); + finalizeQueryResult(pRuntimeEnv); - clearFirstNWindowRes(pRuntimeEnv, pQInfo->groupIndex); - } + // skip offset result rows + pQuery->rec.rows = 0; - // no result generated, abort - if (pQuery->rec.rows == 0 || pRuntimeEnv->groupbyNormalCol) { - break; + // not fill or no result generated during this query + if (pQuery->fillType == TSDB_FILL_NONE || pRuntimeEnv->resultRowInfo.size == 0 || isPointInterpoQuery(pQuery)) { + // all data scanned, the group by normal column can return + int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->resultRowInfo); + if (pQuery->limit.offset > numOfClosed || numOfClosed == 0) { + return; } + initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, (int32_t) pQuery->limit.offset); + copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); doSecondaryArithmeticProcess(pQuery); - - // the offset is handled at prepare stage if no interpolation involved - if (pQuery->fillType == TSDB_FILL_NONE) { - limitResults(pRuntimeEnv); - break; - } else { - taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); - taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata); - numOfFilled = 0; - - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); - if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - limitResults(pRuntimeEnv); - break; - } - - // no result generated yet, continue retrieve data - pQuery->rec.rows = 0; - } - } - - // all data scanned, the group by normal column can return - if (pRuntimeEnv->groupbyNormalCol) { // todo refactor with merge interval time result - // maxOutput <= 0, means current query does not generate any results - int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo); - - if ((pQuery->limit.offset > 0 && pQuery->limit.offset < numOfClosed) || pQuery->limit.offset == 0) { - // skip offset result rows - clearFirstNWindowRes(pRuntimeEnv, (int32_t) pQuery->limit.offset); - - pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - clearFirstNWindowRes(pRuntimeEnv, pQInfo->groupIndex); - doSecondaryArithmeticProcess(pQuery); - limitResults(pRuntimeEnv); - } + limitOperator(pQuery, pQInfo); + } else { + initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, 0); + copyAndFillResult(pQInfo); } } -static void tableQueryImpl(SQInfo *pQInfo) { +void tableQueryImpl(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - if (queryHasRemainResForTableQuery(pRuntimeEnv)) { - - if (pQuery->fillType != TSDB_FILL_NONE) { + if (hasNotReturnedResults(pRuntimeEnv, &pQInfo->groupResInfo)) { + if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { /* * There are remain results that are not returned due to result interpolation * So, we do keep in this procedure instead of launching retrieve procedure for next results. */ - int32_t numOfFilled = 0; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); - + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata); if (pQuery->rec.rows > 0) { - limitResults(pRuntimeEnv); + limitOperator(pQuery, pQInfo); + qDebug("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); + } else { + copyAndFillResult(pQInfo); } - qDebug("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); - return; } else { pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; // always start from 0 - - if (pRuntimeEnv->windowResInfo.size > 0) { - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - clearFirstNWindowRes(pRuntimeEnv, pQInfo->groupIndex); - - if (pQuery->rec.rows > 0) { - qDebug("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); + assert(pRuntimeEnv->resultRowInfo.size > 0); + copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); + doSecondaryArithmeticProcess(pQuery); - // there are not data remains - if (pRuntimeEnv->windowResInfo.size <= 0) { - qDebug("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total); - } + if (pQuery->rec.rows > 0) { + limitOperator(pQuery, pQInfo); + } - return; - } + if (pQuery->rec.rows > 0) { + qDebug("QInfo:%p %" PRId64 " rows returned from group results, total:%" PRId64 "", pQInfo, pQuery->rec.rows, + pQuery->rec.total); + } else { + qDebug("QInfo:%p query over, %" PRId64 " rows are returned", pQInfo, pQuery->rec.total); } } + + return; } // number of points returned during this query @@ -6120,16 +5795,18 @@ static void tableQueryImpl(SQInfo *pQInfo) { assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); SArray* g = GET_TABLEGROUP(pQInfo, 0); + STableQueryInfo* item = taosArrayGetP(g, 0); + pQuery->current = item; // group by normal column, sliding window query, interval query are handled by interval query processor - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); } else if (isFixedOutputQuery(pRuntimeEnv)) { - tableFixedOutputProcess(pQInfo, item); + tableAggregationProcess(pQInfo, item); } else { // diff/add/multiply/subtract/division - assert(pQuery->checkBuffer == 1); - tableMultiOutputProcess(pQInfo, item); + assert(pQuery->checkResultBuf == 1); + tableProjectionProcess(pQInfo, item); } // record the total elapsed time @@ -6137,7 +5814,60 @@ static void tableQueryImpl(SQInfo *pQInfo) { assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); } -static void stableQueryImpl(SQInfo *pQInfo) { +void buildTableBlockDistResult(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pRuntimeEnv->pQuery; + pQuery->pos = 0; + + STableBlockDist *pTableBlockDist = calloc(1, sizeof(STableBlockDist)); + pTableBlockDist->dataBlockInfos = taosArrayInit(512, sizeof(SDataBlockInfo)); + pTableBlockDist->result = (char *)malloc(512); + + TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; + SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; + SSchema blockDistSchema = tGetBlockDistColumnSchema(); + + int64_t startTime = taosGetTimestampUs(); + while (tsdbNextDataBlockWithoutMerge(pQueryHandle)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { + freeTableBlockDist(pTableBlockDist); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + if (pTableBlockDist->firstSeekTimeUs == 0) { + pTableBlockDist->firstSeekTimeUs = taosGetTimestampUs() - startTime; + } + + tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); + taosArrayPush(pTableBlockDist->dataBlockInfos, &blockInfo); + } + if (terrno != TSDB_CODE_SUCCESS) { + freeTableBlockDist(pTableBlockDist); + longjmp(pRuntimeEnv->env, terrno); + } + + pTableBlockDist->numOfRowsInMemTable = tsdbGetNumOfRowsInMemTable(pQueryHandle); + + generateBlockDistResult(pTableBlockDist); + + int type = -1; + assert(pQuery->numOfOutput == 1); + SExprInfo* pExprInfo = pQuery->pExpr1; + for (int32_t j = 0; j < pQuery->numOfOutput; j++) { + if (pExprInfo[j].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + type = blockDistSchema.type; + } + assert(type == TSDB_DATA_TYPE_BINARY); + STR_WITH_SIZE_TO_VARSTR(pQuery->sdata[j]->data, pTableBlockDist->result, (VarDataLenT)strlen(pTableBlockDist->result)); + } + + freeTableBlockDist(pTableBlockDist); + + pQuery->rec.rows = 1; + setQueryStatus(pQuery, QUERY_COMPLETED); + return; +} + +void stableQueryImpl(SQInfo *pQInfo) { SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->rec.rows = 0; @@ -6145,12 +5875,10 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyColumn))) { multiTableQueryProcess(pQInfo); } else { - assert((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || - isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); - + assert(pQuery->checkResultBuf == 1 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyColumn); sequentialTableProcess(pQInfo); } @@ -6164,7 +5892,10 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { if (pExprMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { return TSDB_TBNAME_COLUMN_INDEX; + } else if (pExprMsg->colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + return TSDB_BLOCK_DIST_COLUMN_INDEX; } + while(j < pQueryMsg->numOfTags) { if (pExprMsg->colInfo.colId == pTagCols[j].colId) { @@ -6185,13 +5916,13 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } } - assert(0); - return -1; + + return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value } bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg, pTagCols); - return j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags; + return j != INT32_MIN; } static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { @@ -6218,12 +5949,14 @@ static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { return true; } -static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg) { +static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg, SColumnInfo* pTagCols) { int32_t numOfTotal = pQueryMsg->numOfCols + pQueryMsg->numOfTags; if (pQueryMsg->numOfCols < 0 || pQueryMsg->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pQueryMsg, pQueryMsg->numOfCols, pQueryMsg->numOfTags); return false; - } else if (numOfTotal == 0) { + } + + if (numOfTotal == 0) { for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { SSqlFuncMsg* pFuncMsg = pExprMsg[i]; @@ -6237,6 +5970,12 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx } } + for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + if (!validateExprColumnInfo(pQueryMsg, pExprMsg[i], pTagCols)) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } + return true; } @@ -6267,8 +6006,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, SSqlFuncMsg ***pSecStageExpr, - char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { int32_t code = TSDB_CODE_SUCCESS; if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -6284,7 +6022,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); - pQueryMsg->tableLimit = htobe64(pQueryMsg->tableLimit); + pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); pQueryMsg->order = htons(pQueryMsg->order); pQueryMsg->orderColId = htons(pQueryMsg->orderColId); @@ -6300,7 +6038,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); + pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); + pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); + pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); // query msg safety check if (!validateQueryMsg(pQueryMsg)) { @@ -6317,7 +6058,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pColInfo->bytes = htons(pColInfo->bytes); pColInfo->numOfFilters = htons(pColInfo->numOfFilters); - assert(pColInfo->type >= TSDB_DATA_TYPE_BOOL && pColInfo->type <= TSDB_DATA_TYPE_NCHAR); + if (!isValidDataType(pColInfo->type)) { + qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; + } int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { @@ -6357,8 +6102,8 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); - if (*pExpr == NULL) { + param->pExprMsg = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (param->pExprMsg == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } @@ -6366,7 +6111,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - (*pExpr)[i] = pExprMsg; + param->pExprMsg[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); @@ -6395,26 +6140,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; } - if (!validateQuerySourceCols(pQueryMsg, *pExpr)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - if (pQueryMsg->secondStageOutput) { pExprMsg = (SSqlFuncMsg *)pMsg; - *pSecStageExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); - + param->pSecExprMsg = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); + for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { - (*pSecStageExpr)[i] = pExprMsg; + param->pSecExprMsg[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); @@ -6442,37 +6178,33 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; } } - pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); + pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - *groupbyCols = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); - if (*groupbyCols == NULL) { + param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); + if (param->pGroupColIndex == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - (*groupbyCols)[i].colId = *(int16_t *)pMsg; - pMsg += sizeof((*groupbyCols)[i].colId); + param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colId); - (*groupbyCols)[i].colIndex = *(int16_t *)pMsg; - pMsg += sizeof((*groupbyCols)[i].colIndex); + param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colIndex); - (*groupbyCols)[i].flag = *(int16_t *)pMsg; - pMsg += sizeof((*groupbyCols)[i].flag); + param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].flag); - memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name)); - pMsg += tListLen((*groupbyCols)[i].name); + memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); + pMsg += tListLen(param->pGroupColIndex[i].name); } pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); @@ -6492,8 +6224,8 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } if (pQueryMsg->numOfTags > 0) { - (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); - if (*tagCols == NULL) { + param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (param->pTagColumnInfo == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } @@ -6506,35 +6238,55 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pTagCol->type = htons(pTagCol->type); pTagCol->numOfFilters = 0; - (*tagCols)[i] = *pTagCol; + param->pTagColumnInfo[i] = *pTagCol; pMsg += sizeof(SColumnInfo); } } // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryMsg->tagCondLen); - - if (*tagCond == NULL) { + param->tagCond = calloc(1, pQueryMsg->tagCondLen); + if (param->tagCond == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; - } - memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); + + memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } - if (*pMsg != 0) { - size_t len = strlen(pMsg) + 1; + if (pQueryMsg->prevResultLen > 0) { + param->prevResult = calloc(1, pQueryMsg->prevResultLen); + if (param->prevResult == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); + pMsg += pQueryMsg->prevResultLen; + } - *tbnameCond = malloc(len); - if (*tbnameCond == NULL) { + if (pQueryMsg->tbnameCondLen > 0) { + param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); + if (param->tbnameCond == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } - strcpy(*tbnameCond, pMsg); - pMsg += len; + strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); + pMsg += pQueryMsg->tbnameCondLen; + } + + //skip ts buf + if ((pQueryMsg->tsOffset + pQueryMsg->tsLen) > 0) { + pMsg = (char *)pQueryMsg + pQueryMsg->tsOffset + pQueryMsg->tsLen; + } + + param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); + + if (!validateQuerySourceCols(pQueryMsg, param->pExprMsg, param->pTagColumnInfo)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; } qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " @@ -6543,17 +6295,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); + qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); return TSDB_CODE_SUCCESS; _cleanup: - tfree(*pExpr); - taosArrayDestroy(*pTableIdList); - *pTableIdList = NULL; - tfree(*tbnameCond); - tfree(*groupbyCols); - tfree(*tagCols); - tfree(*tagCond); - + freeParam(param); return code; } @@ -6578,7 +6324,7 @@ static int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTable return TSDB_CODE_SUCCESS; } -static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg, +int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg, SColumnInfo* pTagCols) { *pExprInfo = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -6608,9 +6354,13 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num } type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypeDesc[type].nSize; + bytes = tDataTypes[type].bytes; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column - SSchema s = tGetTableNameColumnSchema(); + SSchema* s = tGetTbnameColumnSchema(); + type = s->type; + bytes = s->bytes; + } else if (pExprs[i].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SSchema s = tGetBlockDistColumnSchema(); type = s.type; bytes = s.bytes; } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX) { @@ -6625,17 +6375,25 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num } } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); + if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { + if (j < TSDB_BLOCK_DIST_COLUMN_INDEX || j >= pQueryMsg->numOfTags) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } else { + if (j < PRIMARYKEY_TIMESTAMP_COL_INDEX || j >= pQueryMsg->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; type = pCol->type; bytes = pCol->bytes; } else { - SSchema s = tGetTableNameColumnSchema(); + SSchema* s = tGetTbnameColumnSchema(); - type = s.type; - bytes = s.bytes; + type = s->type; + bytes = s->bytes; } } @@ -6649,6 +6407,7 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExprs[i].bytes; } + assert(isValidDataType(pExprs[i].type)); } @@ -6660,7 +6419,7 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); if (j < 0 || j >= pQueryMsg->numOfCols) { - assert(0); + return TSDB_CODE_QRY_INVALID_MSG; } else { SColumnInfo *pCol = &pQueryMsg->colList[j]; int32_t ret = @@ -6675,7 +6434,7 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num return TSDB_CODE_SUCCESS; } -static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { +SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; } @@ -6734,55 +6493,18 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; - if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { qError("QInfo:%p invalid filter info", pQInfo); return TSDB_CODE_QRY_INVALID_MSG; } - int16_t type = pQuery->colList[i].type; - int16_t bytes = pQuery->colList[i].bytes; - - // todo refactor - __filter_func_t *rangeFilterArray = getRangeFilterFuncArray(type); - __filter_func_t *filterArray = getValueFilterFuncArray(type); - - if (rangeFilterArray == NULL && filterArray == NULL) { - qError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type); + pSingleColFilter->fp = getFilterOperator(lower, upper); + if (pSingleColFilter->fp == NULL) { + qError("QInfo:%p invalid filter info", pQInfo); return TSDB_CODE_QRY_INVALID_MSG; } - if ((lower == TSDB_RELATION_GREATER_EQUAL || lower == TSDB_RELATION_GREATER) && - (upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) { - assert(rangeFilterArray != NULL); - if (lower == TSDB_RELATION_GREATER_EQUAL) { - if (upper == TSDB_RELATION_LESS_EQUAL) { - pSingleColFilter->fp = rangeFilterArray[4]; - } else { - pSingleColFilter->fp = rangeFilterArray[2]; - } - } else { - if (upper == TSDB_RELATION_LESS_EQUAL) { - pSingleColFilter->fp = rangeFilterArray[3]; - } else { - pSingleColFilter->fp = rangeFilterArray[1]; - } - } - } else { // set callback filter function - assert(filterArray != NULL); - if (lower != TSDB_RELATION_INVALID) { - pSingleColFilter->fp = filterArray[lower]; - - if (upper != TSDB_RELATION_INVALID) { - qError("pQInfo:%p failed to get filter function, invalid filter condition: %d", pQInfo, type); - return TSDB_CODE_QRY_INVALID_MSG; - } - } else { - pSingleColFilter->fp = filterArray[upper]; - } - } - assert(pSingleColFilter->fp != NULL); - pSingleColFilter->bytes = bytes; + pSingleColFilter->bytes = pQuery->colList[i].bytes; } j++; @@ -6824,20 +6546,18 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } - assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); + assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX || pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX); } } } -static void freeQInfo(SQInfo *pQInfo); - static void calResultBufSize(SQuery* pQuery) { const int32_t RESULT_MSG_MIN_SIZE = 1024 * (1024 + 512); // bytes const int32_t RESULT_MSG_MIN_ROWS = 8192; const float RESULT_THRESHOLD_RATIO = 0.85f; if (isProjQuery(pQuery)) { - int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; + int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->resultRowSize; if (numOfRes < RESULT_MSG_MIN_ROWS) { numOfRes = RESULT_MSG_MIN_ROWS; } @@ -6850,8 +6570,8 @@ static void calResultBufSize(SQuery* pQuery) { } } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery) { +SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -6885,23 +6605,35 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->fillType = pQueryMsg->fillType; pQuery->numOfTags = pQueryMsg->numOfTags; pQuery->tagColList = pTagCols; + pQuery->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; + pQuery->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQuery->colList == NULL) { goto _cleanup; } - int32_t srcSize = 0; + pQuery->srcRowSize = 0; + pQuery->maxSrcColumnSize = 0; for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i] = pQueryMsg->colList[i]; - pQuery->colList[i].filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); - srcSize += pQuery->colList[i].bytes; + pQuery->colList[i].filters = tFilterInfoDup(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); + + pQuery->srcRowSize += pQuery->colList[i].bytes; + if (pQuery->maxSrcColumnSize < pQuery->colList[i].bytes) { + pQuery->maxSrcColumnSize = pQuery->colList[i].bytes; + } } // calculate the result row size for (int16_t col = 0; col < numOfOutput; ++col) { assert(pExprs[col].bytes > 0); - pQuery->rowSize += pExprs[col].bytes; + pQuery->resultRowSize += pExprs[col].bytes; + + // keep the tag length + if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { + pQuery->tagLen += pExprs[col].bytes; + } } doUpdateExprColumnIndex(pQuery); @@ -6949,26 +6681,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou size_t numOfGroups = 0; if (pTableGroupInfo->pGroupList != NULL) { numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); + STableGroupInfo* pTableqinfo = &pQInfo->tableqinfoGroupInfo; - pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); - pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; - pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - - pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); - pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); - - pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); - pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv)); - pQInfo->runtimeEnv.prevRow = malloc(POINTER_BYTES * pQuery->numOfCols + srcSize); - - char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pQInfo->runtimeEnv.prevRow; - pQInfo->runtimeEnv.prevRow[0] = start; - - for(int32_t i = 1; i < pQuery->numOfCols; ++i) { - pQInfo->runtimeEnv.prevRow[i] = pQInfo->runtimeEnv.prevRow[i - 1] + pQuery->colList[i-1].bytes; + pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); + pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; + pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); @@ -6980,6 +6697,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; + pQInfo->sql = sql; pthread_mutex_init(&pQInfo->lock, NULL); tsem_init(&pQInfo->ready, 0, 0); @@ -6988,6 +6706,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou changeExecuteScanOrder(pQInfo, pQueryMsg, stableQuery); pQInfo->runtimeEnv.queryWindowIdentical = true; + bool groupByCol = isGroupbyColumn(pQuery->pGroupbyExpr); + STimeWindow window = pQuery->window; int32_t index = 0; @@ -7011,7 +6731,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); + STableQueryInfo* item = createTableQueryInfo(pQuery, info->pTable, groupByCol, window, buf); if (item == NULL) { goto _cleanup; } @@ -7027,6 +6747,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou colIdCheck(pQuery); + // todo refactor + pQInfo->runtimeEnv.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX); + qDebug("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); return pQInfo; @@ -7043,7 +6766,8 @@ _cleanup_query: for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(&pExprInfo->pExpr, NULL); + tExprTreeDestroy(pExprInfo->pExpr, NULL); + pExprInfo->pExpr = NULL; } } @@ -7054,7 +6778,7 @@ _cleanup: return NULL; } -static bool isValidQInfo(void *param) { +bool isValidQInfo(void *param) { SQInfo *pQInfo = (SQInfo *)param; if (pQInfo == NULL) { return false; @@ -7068,21 +6792,26 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) { +int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, SQueryParam* param, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - STSBuf *pTSBuf = NULL; + STSBuf *pTsBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; - pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); + pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); - tsBufResetPos(pTSBuf); - bool ret = tsBufNextPos(pTSBuf); + tsBufResetPos(pTsBuf); + bool ret = tsBufNextPos(pTsBuf); UNUSED(ret); } - + + SArray* prevResult = NULL; + if (pQueryMsg->prevResultLen > 0) { + prevResult = interResFromBinary(param->prevResult, pQueryMsg->prevResultLen); + } + pQuery->precision = tsdbGetCfg(tsdb)->precision; if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || @@ -7091,6 +6820,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ pQuery->window.ekey, pQuery->order.order); setQueryStatus(pQuery, QUERY_COMPLETED); pQInfo->tableqinfoGroupInfo.numOfTables = 0; + // todo free memory return TSDB_CODE_SUCCESS; } @@ -7101,7 +6831,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ } // filter the qualified - if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTsBuf, prevResult, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -7113,7 +6843,7 @@ _error: return code; } -static void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { if (pFilter == NULL || numOfFilters == 0) { return; } @@ -7159,7 +6889,7 @@ static void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { for (int32_t i = 0; i < numOfExpr; ++i) { if (pExprInfo[i].pExpr != NULL) { - tExprNodeDestroy(pExprInfo[i].pExpr, NULL); + tExprTreeDestroy(pExprInfo[i].pExpr, NULL); } } @@ -7167,7 +6897,7 @@ static void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { return NULL; } -static void freeQInfo(SQInfo *pQInfo) { +void freeQInfo(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; } @@ -7175,7 +6905,6 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p start to free QInfo", pQInfo); releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables); - teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -7223,9 +6952,13 @@ static void freeQInfo(SQInfo *pQInfo) { doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); tfree(pQInfo->pBuf); + tfree(pQInfo->sql); + tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosHashCleanup(pQInfo->arrTableIdInfo); + taosArrayDestroy(pQInfo->groupResInfo.pRows); + pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -7233,7 +6966,7 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQInfo); } -static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { +size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; /* @@ -7241,51 +6974,55 @@ static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { * the returned row size is equalled to 1 * TODO handle the case that the file is too large to send back one time */ - if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fstat; - if (stat(pQuery->sdata[0]->data, &fstat) == 0) { - *numOfRows = fstat.st_size; - return fstat.st_size; + if (isTsCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fStat; + FILE *f = *(FILE **)pQuery->sdata[0]->data; + if ((f != NULL) && (fstat(fileno(f), &fStat) == 0)) { + *numOfRows = fStat.st_size; + return fStat.st_size; } else { - qError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + qError("QInfo:%p failed to get file info, file:%p, reason:%s", pQInfo, f, strerror(errno)); return 0; } } else { - return (size_t)(pQuery->rowSize * (*numOfRows)); + return (size_t)(pQuery->resultRowSize * (*numOfRows)); } } -static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // the remained number of retrieved rows, not the interpolated result SQuery *pQuery = pQInfo->runtimeEnv.pQuery; // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + if (isTsCompQuery(pQuery)) { + + FILE *f = *(FILE **)pQuery->sdata[0]->data; // TODO refactor // make sure file exist - if (FD_VALID(fd)) { - uint64_t s = lseek(fd, 0, SEEK_END); + if (f) { + off_t s = lseek(fileno(f), 0, SEEK_END); - qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); - if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, (uint32_t) s); + qDebug("QInfo:%p ts comp data return, file:%p, size:%"PRId64, pQInfo, f, (uint64_t)s); + if (fseek(f, 0, SEEK_SET) >= 0) { + size_t sz = fread(data, 1, s, f); if(sz < s) { // todo handle error + qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s); assert(0); } } else { UNUSED(s); + qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno)); + assert(0); } - close(fd); - unlink(pQuery->sdata[0]->data); - } else { - // todo return the error code to client and handle invalid fd - qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, - pQuery->sdata[0]->data, strerror(errno)); - if (fd != -1) { - close(fd); + if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { + qDump(data, s); + + assert(0); } + + fclose(f); + *(FILE **)pQuery->sdata[0]->data = NULL; } // all data returned, set query over @@ -7307,392 +7044,40 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { return TSDB_CODE_SUCCESS; } -typedef struct SQueryMgmt { - SCacheObj *qinfoPool; // query handle pool - int32_t vgId; - bool closed; - pthread_mutex_t lock; -} SQueryMgmt; - -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) { - assert(pQueryMsg != NULL && tsdb != NULL); - - int32_t code = TSDB_CODE_SUCCESS; - - char *tagCond = NULL; - char *tbnameCond = NULL; - SArray *pTableIdList = NULL; - SSqlFuncMsg **pExprMsg = NULL; - SSqlFuncMsg **pSecExprMsg = NULL; - SExprInfo *pExprs = NULL; - SExprInfo *pSecExprs = NULL; - - SColIndex *pGroupColIndex = NULL; - SColumnInfo *pTagColumnInfo = NULL; - SSqlGroupbyExpr *pGroupbyExpr = NULL; - - code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->numOfOutput, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (pSecExprMsg != NULL) { - if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &pSecExprs, pSecExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code); - if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %" PRIzu " tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%" PRIzu ", elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); - } - - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery); - - pExprs = NULL; - pSecExprs = NULL; - pGroupbyExpr = NULL; - pTagColumnInfo = NULL; - - if ((*pQInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - - code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); - -_over: - free(tagCond); - free(tbnameCond); - free(pGroupColIndex); - - if (pGroupbyExpr != NULL) { - taosArrayDestroy(pGroupbyExpr->columnInfo); - free(pGroupbyExpr); - } - - free(pTagColumnInfo); - free(pExprs); - free(pSecExprs); - - free(pExprMsg); - free(pSecExprMsg); - - taosArrayDestroy(pTableIdList); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->colList + i; - freeColumnFilterInfo(column->filters, column->numOfFilters); - } - - //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; - if (code != TSDB_CODE_SUCCESS) { - *pQInfo = NULL; - } - - // if failed to add ref for all tables in this query, abort current query - return code; -} - -void qDestroyQueryInfo(qinfo_t qHandle) { - SQInfo* pQInfo = (SQInfo*) qHandle; - if (!isValidQInfo(pQInfo)) { - return; - } - - qDebug("QInfo:%p query completed", pQInfo); - queryCostStatis(pQInfo); // print the query cost summary - freeQInfo(pQInfo); -} - -static bool doBuildResCheck(SQInfo* pQInfo) { +bool doBuildResCheck(SQInfo* pQInfo) { bool buildRes = false; pthread_mutex_lock(&pQInfo->lock); pQInfo->dataReady = QUERY_RESULT_READY; - buildRes = (pQInfo->rspContext != NULL); + buildRes = needBuildResAfterQueryComplete(pQInfo); // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is // put into task to be executed. - assert(pQInfo->owner == taosGetPthreadId()); + assert(pQInfo->owner == taosGetSelfPthreadId()); pQInfo->owner = 0; pthread_mutex_unlock(&pQInfo->lock); + // used in retrieve blocking model. tsem_post(&pQInfo->ready); return buildRes; } -bool qTableQuery(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); - int64_t threadId = taosGetPthreadId(); - - int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:%p qhandle is now executed by thread:%p", pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; - return false; - } - - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:%p it is already killed, abort", pQInfo); - return doBuildResCheck(pQInfo); - } - - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:%p no table exists for query, abort", pQInfo); - setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); - } - - // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); - if (ret != TSDB_CODE_SUCCESS) { - pQInfo->code = ret; - qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); - } - - qDebug("QInfo:%p query task is launched", pQInfo); - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { - assert(pQInfo->runtimeEnv.pQueryHandle == NULL); - buildTagQueryResult(pQInfo); - } else if (pQInfo->runtimeEnv.stableQuery) { - stableQueryImpl(pQInfo); - } else { - tableQueryImpl(pQInfo); - } - - SQuery* pQuery = pRuntimeEnv->pQuery; - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:%p query is killed", pQInfo); - } else if (pQuery->rec.rows == 0) { - qDebug("QInfo:%p over, %" PRIzu " tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total); - } else { - qDebug("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", - pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); - } - - return doBuildResCheck(pQInfo); -} - -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - qError("QInfo:%p invalid qhandle", pQInfo); - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - *buildRes = false; - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:%p query is killed, code:0x%08x", pQInfo, pQInfo->code); - return pQInfo->code; - } - - int32_t code = TSDB_CODE_SUCCESS; - -#if _NON_BLOCKING_RETRIEVE - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - pthread_mutex_lock(&pQInfo->lock); - assert(pQInfo->rspContext == NULL); - - if (pQInfo->dataReady == QUERY_RESULT_READY) { - *buildRes = true; - qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, - pQInfo->code); - } else { - *buildRes = false; - qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo); - pQInfo->rspContext = pRspContext; - assert(pQInfo->rspContext != NULL); - } - - code = pQInfo->code; - pthread_mutex_unlock(&pQInfo->lock); -#else - tsem_wait(&pQInfo->ready); - *buildRes = true; - code = pQInfo->code; -#endif - - return code; -} - -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.rows); - - size += sizeof(int32_t); - size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo); - - *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); - - // todo proper handle failed to allocate memory, - // current solution only avoid crash, but cannot return error code to client - *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); - if (*pRsp == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - (*pRsp)->numOfRows = htonl((int32_t)pQuery->rec.rows); - - if (pQInfo->code == TSDB_CODE_SUCCESS) { - (*pRsp)->offset = htobe64(pQuery->limit.offset); - (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); - } else { - (*pRsp)->offset = 0; - (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); - } - - (*pRsp)->precision = htons(pQuery->precision); - if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { - doDumpQueryResult(pQInfo, (*pRsp)->data); - } else { - setQueryStatus(pQuery, QUERY_OVER); - } - - pQInfo->rspContext = NULL; - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - - if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. - *continueExec = false; - (*pRsp)->completed = 1; // notify no more result to client - } else { - *continueExec = true; - qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); - } - - return pQInfo->code; -} - -int32_t qQueryCompleted(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - return IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER); -} - -int32_t qKillQuery(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - setQueryKilled(pQInfo); - - // Wait for the query executing thread being stopped/ - // Once the query is stopped, the owner of qHandle will be cleared immediately. - while (pQInfo->owner != 0) { - taosMsleep(100); +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { + if (val == NULL) { + setNull(output, type, bytes); + return; } - return TSDB_CODE_SUCCESS; -} - -static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - if (val == NULL) { - setVardataNull(output, type); - } else { - memcpy(output, val, varDataTLen(val)); - } + memcpy(output, val, varDataTLen(val)); } else { - if (val == NULL) { - setNull(output, type, bytes); - } else { // todo here stop will cause client crash - memcpy(output, val, bytes); - } + memcpy(output, val, bytes); } } -static void buildTagQueryResult(SQInfo* pQInfo) { +void buildTagQueryResult(SQInfo* pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -7771,7 +7156,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { qDebug("QInfo:%p create count(tbname) query, res:%d rows:1", pQInfo, count); } else { // return only the tags|table name etc. count = 0; - SSchema tbnameSchema = tGetTableNameColumnSchema(); + SSchema* tbnameSchema = tGetTbnameColumnSchema(); int32_t maxNumOfTables = (int32_t)pQuery->rec.capacity; if (pQuery->limit.limit >= 0 && pQuery->limit.limit < pQuery->rec.capacity) { @@ -7799,11 +7184,11 @@ static void buildTagQueryResult(SQInfo* pQInfo) { } if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - bytes = tbnameSchema.bytes; - type = tbnameSchema.type; + bytes = tbnameSchema->bytes; + type = tbnameSchema->type; data = tsdbGetTableName(item->pTable); - dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes; + dst = pQuery->sdata[j]->data + count * tbnameSchema->bytes; } else { type = pExprInfo[j].type; bytes = pExprInfo[j].bytes; @@ -7835,15 +7220,15 @@ static int64_t getQuerySupportBufSize(size_t numOfTables) { int32_t checkForQueryBuf(size_t numOfTables) { int64_t t = getQuerySupportBufSize(numOfTables); - if (tsQueryBufferSize < 0) { + if (tsQueryBufferSizeBytes < 0) { return TSDB_CODE_SUCCESS; - } else if (tsQueryBufferSize > 0) { + } else if (tsQueryBufferSizeBytes > 0) { while(1) { - int64_t s = tsQueryBufferSize; + int64_t s = tsQueryBufferSizeBytes; int64_t remain = s - t; if (remain >= 0) { - if (atomic_val_compare_exchange_64(&tsQueryBufferSize, s, remain) == s) { + if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { return TSDB_CODE_SUCCESS; } } else { @@ -7857,156 +7242,12 @@ int32_t checkForQueryBuf(size_t numOfTables) { } void releaseQueryBuf(size_t numOfTables) { - if (tsQueryBufferSize <= 0) { + if (tsQueryBufferSizeBytes < 0) { return; } int64_t t = getQuerySupportBufSize(numOfTables); // restore value is not enough buffer available - atomic_add_fetch_64(&tsQueryBufferSize, t); -} - -void* qGetResultRetrieveMsg(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - assert(pQInfo != NULL); - - return pQInfo->rspContext; -} - -void freeqinfoFn(void *qhandle) { - void** handle = qhandle; - if (handle == NULL || *handle == NULL) { - return; - } - - qKillQuery(*handle); - qDestroyQueryInfo(*handle); -} - -void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool - - char cacheName[128] = {0}; - sprintf(cacheName, "qhandle_%d", vgId); - - SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); - if (pQueryMgmt == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pQueryMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); - pQueryMgmt->closed = false; - pQueryMgmt->vgId = vgId; - - pthread_mutex_init(&pQueryMgmt->lock, NULL); - - qDebug("vgId:%d, open querymgmt success", vgId); - return pQueryMgmt; -} - -static void queryMgmtKillQueryFn(void* handle) { - void** fp = (void**)handle; - qKillQuery(*fp); -} - -void qQueryMgmtNotifyClosed(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); - -// pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = true; -// pthread_mutex_unlock(&pQueryMgmt->lock); - - taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); -} - -void qCleanupQueryMgmt(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - int32_t vgId = pQueryMgmt->vgId; - - assert(pQueryMgmt->closed); - - SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; - pQueryMgmt->qinfoPool = NULL; - - taosCacheCleanup(pqinfoPool); - pthread_mutex_destroy(&pQueryMgmt->lock); - tfree(pQueryMgmt); - - qDebug("vgId:%d, queryMgmt cleanup completed", vgId); -} - -void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { - if (pMgmt == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - const int32_t DEFAULT_QHANDLE_LIFE_SPAN = tsShellActivityTimer * 2 * 1000; - - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - qError("QInfo:%p failed to add qhandle into qMgmt, since qMgmt is closed", (void *)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - -// pthread_mutex_lock(&pQueryMgmt->lock); - if (pQueryMgmt->closed) { -// pthread_mutex_unlock(&pQueryMgmt->lock); - qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } else { - TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo; - void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), DEFAULT_QHANDLE_LIFE_SPAN); -// pthread_mutex_unlock(&pQueryMgmt->lock); - - return handle; - } -} - -void** qAcquireQInfo(void* pMgmt, uint64_t _key) { - SQueryMgmt *pQueryMgmt = pMgmt; - - if (pQueryMgmt->closed) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - if (pQueryMgmt->qinfoPool == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } - - TSDB_CACHE_PTR_TYPE key = (TSDB_CACHE_PTR_TYPE)_key; - void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &key, sizeof(TSDB_CACHE_PTR_TYPE)); - if (handle == NULL || *handle == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } else { - return handle; - } + atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } - -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - return NULL; - } - - taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); - return 0; -} - - diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index fa3fe285a857cc0d146e25f52710d0ebc0a69193..a73f38528266bfa70790414a2963eb0a8290d7b9 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,13 +12,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" #include "qExtbuffer.h" +#include "os.h" +#include "qAggMain.h" #include "queryLog.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" -#include "tsqlfunction.h" #include "tulog.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ @@ -362,20 +362,10 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i return (first < second) ? -1 : 1; }; case TSDB_DATA_TYPE_DOUBLE: { - double first = GET_DOUBLE_VAL(f1); - double second = GET_DOUBLE_VAL(f2); - if (first == second) { - return 0; - } - return (first < second) ? -1 : 1; + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); }; case TSDB_DATA_TYPE_FLOAT: { - float first = GET_FLOAT_VAL(f1); - float second = GET_FLOAT_VAL(f2); - if (first == second) { - return 0; - } - return (first < second) ? -1 : 1; + DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); }; case TSDB_DATA_TYPE_BIGINT: { int64_t first = *(int64_t *)f1; diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index ca1203cb17da52b8ffe638e29913b06bdfcade29..bc6376b80743bcad3e1b07f903c970eeb9e5dee2 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -15,9 +15,9 @@ #include "os.h" +#include "qAggMain.h" #include "taosdef.h" #include "taosmsg.h" -#include "tsqlfunction.h" #include "ttype.h" #include "qFill.h" @@ -25,237 +25,8 @@ #include "queryLog.h" #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) - -// there are no duplicated tags in the SFillTagColInfo list -static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) { - int32_t rowsize = 0; - - int32_t k = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity); - - if (TSDB_COL_IS_TAG(pColInfo->flag)) { - bool exists = false; - int32_t index = -1; - for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { - exists = true; - index = j; - break; - } - } - - if (!exists) { - SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.colId; - pSchema->type = pColInfo->col.type; - pSchema->bytes = pColInfo->col.bytes; - - pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); - pColInfo->tagIndex = k; - - k += 1; - } else { - pColInfo->tagIndex = index; - } - } - - rowsize += pColInfo->col.bytes; - } - - assert(k <= pFillInfo->numOfTags); - return rowsize; -} - -SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pCol, void* handle) { - if (fillType == TSDB_FILL_NONE) { - return NULL; - } - - SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); - - taosResetFillInfo(pFillInfo, skey); - - pFillInfo->order = order; - pFillInfo->type = fillType; - pFillInfo->pFillCol = pCol; - pFillInfo->numOfTags = numOfTags; - pFillInfo->numOfCols = numOfCols; - pFillInfo->precision = precision; - pFillInfo->alloc = capacity; - pFillInfo->handle = handle; - - pFillInfo->interval.interval = slidingTime; - pFillInfo->interval.intervalUnit = slidingUnit; - pFillInfo->interval.sliding = slidingTime; - pFillInfo->interval.slidingUnit = slidingUnit; - - pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); - if (numOfTags > 0) { - pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo)); - for (int32_t i = 0; i < numOfTags; ++i) { - pFillInfo->pTags[i].col.colId = -2; // TODO - } - } - - pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); - assert(pFillInfo->rowSize > 0); - - return pFillInfo; -} - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { - pFillInfo->start = startTimestamp; - pFillInfo->currentKey = startTimestamp; - pFillInfo->index = -1; - pFillInfo->numOfRows = 0; - pFillInfo->numOfCurrent = 0; - pFillInfo->numOfTotal = 0; -} - -void* taosDestroyFillInfo(SFillInfo* pFillInfo) { - if (pFillInfo == NULL) { - return NULL; - } - - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); - tfree(pFillInfo->pTags); - - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - tfree(pFillInfo->pData[i]); - } - - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); - - tfree(pFillInfo); - return NULL; -} - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { - if (pFillInfo->type == TSDB_FILL_NONE) { - return; - } - - pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); - } - - pFillInfo->index = 0; - pFillInfo->numOfRows = numOfRows; - - // ensure the space - if (pFillInfo->alloc < numOfRows) { - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes); - assert(tmp != NULL); // todo handle error - - memset(tmp, 0, numOfRows*pFillInfo->pFillCol[i].col.bytes); - pFillInfo->pData[i] = tmp; - } - } -} - -// copy the data into source data buffer -void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) { - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes); - } -} - -void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) { - assert(pFillInfo->numOfRows == pInput->num); - - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - const char* data = pInput->data + pCol->col.offset * pInput->num; - memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes)); - - if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.colId); - memcpy(pTag->tagVal, data, pCol->col.bytes); - } - } -} - -int64_t getNumOfResWithFill(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { - int64_t* tsList = (int64_t*) pFillInfo->pData[0]; - - int32_t numOfRows = taosNumOfRemainRows(pFillInfo); - - TSKEY ekey1 = ekey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); - } - - int64_t numOfRes = -1; - if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. - TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - numOfRes = taosTimeCountInterval( - lastKey, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - assert(numOfRes >= numOfRows); - } else { // reach the end of data - if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { - return 0; - } - numOfRes = taosTimeCountInterval( - ekey1, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - } - - return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; -} - -int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { - if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) { - return 0; - } - - return pFillInfo->numOfRows - pFillInfo->index; -} - #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) -int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) { - double v1 = -1; - double v2 = -1; - - GET_TYPED_DATA(v1, double, type, point1->val); - GET_TYPED_DATA(v2, double, type, point2->val); - - double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); - - switch(type) { - case TSDB_DATA_TYPE_TINYINT: *(int8_t*) point->val = (int8_t) r;break; - case TSDB_DATA_TYPE_SMALLINT: *(int16_t*) point->val = (int16_t) r;break; - case TSDB_DATA_TYPE_INT: *(int32_t*) point->val = (int32_t) r;break; - case TSDB_DATA_TYPE_BIGINT: *(int64_t*) point->val = (int64_t) r;break; - case TSDB_DATA_TYPE_DOUBLE: *(double*) point->val = (double) r;break; - case TSDB_DATA_TYPE_FLOAT: *(float*) point->val = (float) r;break; - default: - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows) { for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { SFillColInfo* pCol = &pFillInfo->pFillCol[j]; @@ -298,7 +69,23 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr // set the other values if (pFillInfo->type == TSDB_FILL_PREV) { char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next; - + + if (p != NULL) { + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; + } + + char* output = elePtrAt(data[i]->data, pCol->col.bytes, index); + assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); + } + } else { // no prev value yet, set the value for NULL + setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); + } + } else if (pFillInfo->type == TSDB_FILL_NEXT) { + char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; + if (p != NULL) { for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; @@ -323,7 +110,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr int16_t type = pCol->col.type; int16_t bytes = pCol->col.bytes; - + char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, index); if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { setNull(val1, pCol->col.type, bytes); @@ -333,7 +120,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset}; point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; - taosGetLinearInterpolationVal(type, &point1, &point2, &point); + taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); } } else { setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); @@ -359,7 +146,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { if (*next != NULL) { return; } - + *next = calloc(1, pFillInfo->rowSize); for (int i = 1; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; @@ -393,9 +180,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou while (pFillInfo->numOfCurrent < outputRows) { int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index]; + // set the next value for interpolation if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) { - /* set the next value for interpolation */ initBeforeAfterDataBuf(pFillInfo, next); copyCurrentRowIntoBuf(pFillInfo, srcData, *next); } @@ -449,7 +236,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->precision); + pFillInfo->interval.slidingUnit, pFillInfo->precision); pFillInfo->index += 1; pFillInfo->numOfCurrent += 1; } @@ -468,7 +255,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou return pFillInfo->numOfCurrent; } -static int64_t fillExternalResults(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) { +static int64_t appendFilledResult(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) { /* * These data are generated according to fill strategy, since the current timestamp is out of the time window of * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. @@ -484,15 +271,234 @@ static int64_t fillExternalResults(SFillInfo* pFillInfo, tFilePage** output, int return resultCapacity; } +// there are no duplicated tags in the SFillTagColInfo list +static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) { + int32_t rowsize = 0; + + int32_t k = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; + pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity); + + if (TSDB_COL_IS_TAG(pColInfo->flag)) { + bool exists = false; + int32_t index = -1; + for (int32_t j = 0; j < k; ++j) { + if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { + exists = true; + index = j; + break; + } + } + + if (!exists) { + SSchema* pSchema = &pFillInfo->pTags[k].col; + pSchema->colId = pColInfo->col.colId; + pSchema->type = pColInfo->col.type; + pSchema->bytes = pColInfo->col.bytes; + + pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); + pColInfo->tagIndex = k; + + k += 1; + } else { + pColInfo->tagIndex = index; + } + } + + rowsize += pColInfo->col.bytes; + } + + assert(k <= pFillInfo->numOfTags); + return rowsize; +} + +static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { + if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) { + return 0; + } + + return pFillInfo->numOfRows - pFillInfo->index; +} + +SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, + int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, + SFillColInfo* pCol, void* handle) { + if (fillType == TSDB_FILL_NONE) { + return NULL; + } + + SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); + taosResetFillInfo(pFillInfo, skey); + + pFillInfo->order = order; + pFillInfo->type = fillType; + pFillInfo->pFillCol = pCol; + pFillInfo->numOfTags = numOfTags; + pFillInfo->numOfCols = numOfCols; + pFillInfo->precision = precision; + pFillInfo->alloc = capacity; + pFillInfo->handle = handle; + + pFillInfo->interval.interval = slidingTime; + pFillInfo->interval.intervalUnit = slidingUnit; + pFillInfo->interval.sliding = slidingTime; + pFillInfo->interval.slidingUnit = slidingUnit; + + pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); + if (numOfTags > 0) { + pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo)); + for (int32_t i = 0; i < numOfTags; ++i) { + pFillInfo->pTags[i].col.colId = -2; // TODO + } + } + + pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); + assert(pFillInfo->rowSize > 0); + + return pFillInfo; +} + +void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { + pFillInfo->start = startTimestamp; + pFillInfo->currentKey = startTimestamp; + pFillInfo->index = -1; + pFillInfo->numOfRows = 0; + pFillInfo->numOfCurrent = 0; + pFillInfo->numOfTotal = 0; +} + +void* taosDestroyFillInfo(SFillInfo* pFillInfo) { + if (pFillInfo == NULL) { + return NULL; + } + + tfree(pFillInfo->prevValues); + tfree(pFillInfo->nextValues); + tfree(pFillInfo->pTags); + + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + tfree(pFillInfo->pData[i]); + } + + tfree(pFillInfo->pData); + tfree(pFillInfo->pFillCol); + + tfree(pFillInfo); + return NULL; +} + +void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { + if (pFillInfo->type == TSDB_FILL_NONE) { + return; + } + + pFillInfo->end = endKey; + if (!FILL_IS_ASC_FILL(pFillInfo)) { + pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); + } + + pFillInfo->index = 0; + pFillInfo->numOfRows = numOfRows; + + // ensure the space + if (pFillInfo->alloc < numOfRows) { + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes); + assert(tmp != NULL); // todo handle error + + memset(tmp, 0, numOfRows*pFillInfo->pFillCol[i].col.bytes); + pFillInfo->pData[i] = tmp; + } + } +} + +// copy the data into source data buffer +void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) { + for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes); + } +} + +void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) { + assert(pFillInfo->numOfRows == pInput->num); + + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + + const char* data = pInput->data + pCol->col.offset * pInput->num; + memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes)); + + if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer + SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; + assert (pTag->col.colId == pCol->col.colId); + memcpy(pTag->tagVal, data, pCol->col.bytes); + } + } +} + +bool taosFillHasMoreResults(SFillInfo* pFillInfo) { + return taosNumOfRemainRows(pFillInfo) > 0; +} + +int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { + int64_t* tsList = (int64_t*) pFillInfo->pData[0]; + + int32_t numOfRows = taosNumOfRemainRows(pFillInfo); + + TSKEY ekey1 = ekey; + if (!FILL_IS_ASC_FILL(pFillInfo)) { + pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); + } + + int64_t numOfRes = -1; + if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. + TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; + numOfRes = taosTimeCountInterval( + lastKey, + pFillInfo->currentKey, + pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, + pFillInfo->precision); + numOfRes += 1; + assert(numOfRes >= numOfRows); + } else { // reach the end of data + if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || + (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { + return 0; + } + numOfRes = taosTimeCountInterval( + ekey1, + pFillInfo->currentKey, + pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, + pFillInfo->precision); + numOfRes += 1; + } + + return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; +} + +int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) { + double v1 = -1, v2 = -1; + GET_TYPED_DATA(v1, double, inputType, point1->val); + GET_TYPED_DATA(v2, double, inputType, point2->val); + + double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); + SET_TYPED_DATA(point->val, outputType, r); + + return TSDB_CODE_SUCCESS; +} + int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) { int32_t remain = taosNumOfRemainRows(pFillInfo); - int64_t numOfRes = getNumOfResWithFill(pFillInfo, pFillInfo->end, capacity); + int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); assert(numOfRes <= capacity); // no data existed for fill operation now, append result according to the fill strategy if (remain == 0) { - fillExternalResults(pFillInfo, output, numOfRes); + appendFilledResult(pFillInfo, output, numOfRes); } else { fillResultImpl(pFillInfo, output, (int32_t) numOfRes); assert(numOfRes == pFillInfo->numOfCurrent); diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index b6050dddd8fecc8f0a0b764313efcbe2a630113a..884f7e653f0ef05b3f7eb0318ac1e6342b9a539e 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -19,197 +19,174 @@ #include "qExecutor.h" #include "taosmsg.h" #include "tcompare.h" -#include "tsqlfunction.h" +#include "ttype.h" -bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval < pFilter->filterInfo.upperBndi); -} - -bool less_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval < pFilter->filterInfo.upperBndi); -} - -bool less_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval < pFilter->filterInfo.upperBndi); -} - -bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval < pFilter->filterInfo.upperBndi); -} - -bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval < pFilter->filterInfo.upperBndd); -} - -bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)minval < pFilter->filterInfo.upperBndd); -} - -////////////////////////////////////////////////////////////////// -bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)maxval > pFilter->filterInfo.lowerBndd); -} - -bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)maxval > pFilter->filterInfo.lowerBndd); -} -///////////////////////////////////////////////////////////////////// +#define FLT_COMPAR_TOL_FACTOR 4 +#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)) +#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y))) +#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y))) +#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) +#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) -bool lessEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi); -} - -bool lessEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi); -} - -bool lessEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi); -} - -bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi); -} - -bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->filterInfo.upperBndd); -} - -bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)minval <= pFilter->filterInfo.upperBndd); -} - -////////////////////////////////////////////////////////////////////////// -bool largeEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool largeEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool largeEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)maxval >= pFilter->filterInfo.lowerBndd); -} - -bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)maxval >= pFilter->filterInfo.lowerBndd); -} +bool lessOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { + SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo; -//////////////////////////////////////////////////////////////////////// - -bool equal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int8_t *)minval == *(int8_t *)maxval) { - return (*(int8_t *)minval == pFilter->filterInfo.lowerBndi); - } else { /* range filter */ - assert(*(int8_t *)minval < *(int8_t *)maxval); - - return *(int8_t *)minval <= pFilter->filterInfo.lowerBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi; - } -} - -bool equal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int16_t *)minval == *(int16_t *)maxval) { - return (*(int16_t *)minval == pFilter->filterInfo.lowerBndi); - } else { /* range filter */ - assert(*(int16_t *)minval < *(int16_t *)maxval); - - return *(int16_t *)minval <= pFilter->filterInfo.lowerBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi; + switch(type) { + case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval < pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_FLOAT: return FLT_LESS(*(float*)minval, pFilter->filterInfo.upperBndd); + case TSDB_DATA_TYPE_DOUBLE: return (*(double *)minval < pFilterInfo->upperBndd); + default: + return false; } } -bool equal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int32_t *)minval == *(int32_t *)maxval) { - return (*(int32_t *)minval == pFilter->filterInfo.lowerBndi); - } else { /* range filter */ - assert(*(int32_t *)minval < *(int32_t *)maxval); - - return *(int32_t *)minval <= pFilter->filterInfo.lowerBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi; +bool greaterOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval > pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_FLOAT: return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd); + case TSDB_DATA_TYPE_DOUBLE: return (*(double *)maxval > pFilterInfo->lowerBndd); + default: + return false; } } -bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int64_t *)minval == *(int64_t *)maxval) { - return (*(int64_t *)minval == pFilter->filterInfo.lowerBndi); - } else { /* range filter */ - assert(*(int64_t *)minval < *(int64_t *)maxval); - - return *(int64_t *)minval <= pFilter->filterInfo.lowerBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi; - } -} +bool lessEqualOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { + SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo; -bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(float *)minval == *(float *)maxval) { - return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON); - } else { /* range filter */ - assert(*(float *)minval < *(float *)maxval); - return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd; + switch(type) { + case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval <= pFilterInfo->upperBndi); + case TSDB_DATA_TYPE_FLOAT: return FLT_LESSEQUAL(*(float*)minval, pFilterInfo->upperBndd); + case TSDB_DATA_TYPE_DOUBLE: { + if ((fabs(*(double*)minval) - pFilterInfo->upperBndd) <= 2 * DBL_EPSILON) { + return true; + } + + return (*(double *)minval <= pFilterInfo->upperBndd); + } + default: + return false; } } -bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(double *)minval == *(double *)maxval) { - return (*(double *)minval == pFilter->filterInfo.lowerBndd); - } else { /* range filter */ - assert(*(double *)minval < *(double *)maxval); - - return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi; +bool greaterEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval >= pFilterInfo->lowerBndi); + case TSDB_DATA_TYPE_FLOAT: return FLT_GREATEREQUAL(*(float*)maxval, pFilterInfo->lowerBndd); + case TSDB_DATA_TYPE_DOUBLE: { + if (fabs(*(double *)maxval - pFilterInfo->lowerBndd) <= 2 * DBL_EPSILON) { + return true; + } + + return (*(double *)maxval - pFilterInfo->lowerBndd > (2 * DBL_EPSILON)); + } + default: + return false; } } -bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { - // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len != varDataLen(minval)) { +//////////////////////////////////////////////////////////////////////// +bool equalOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { + int64_t minv = -1, maxv = -1; + GET_TYPED_DATA(minv, int64_t, type, minval); + GET_TYPED_DATA(maxv, int64_t, type, maxval); + + if (minv == maxv) { + return minv == pFilterInfo->lowerBndi; + } else { + assert(minv < maxv); + return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv; + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + uint64_t minv = 0, maxv = 0; + GET_TYPED_DATA(minv, uint64_t, type, minval); + GET_TYPED_DATA(maxv, uint64_t, type, maxval); + + if (minv == maxv) { + return minv == pFilterInfo->lowerBndi; + } else { + assert(minv < maxv); + return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv; + } + } else if (IS_FLOAT_TYPE(type)) { + double minv = -1, maxv = -1; + GET_TYPED_DATA(minv, double, type, minval); + GET_TYPED_DATA(maxv, double, type, maxval); + + if (minv == maxv) { + return FLT_EQUAL(minv, pFilterInfo->lowerBndd); + } else { // range filter + assert(minv < maxv); + return minv <= pFilterInfo->lowerBndd && pFilterInfo->lowerBndd <= maxv; + } + } else if (type == TSDB_DATA_TYPE_BINARY) { + // query condition string is greater than the max length of string, not qualified data + if (pFilterInfo->len != varDataLen(minval)) { + return false; + } + + return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) == 0; + } else if (type == TSDB_DATA_TYPE_NCHAR) { + // query condition string is greater than the max length of string, not qualified data + if (pFilterInfo->len != varDataLen(minval)) { + return false; + } + + return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval) / TSDB_NCHAR_SIZE) == 0; + } else { return false; } - - return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) == 0; } -bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { - // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len != varDataLen(minval)) { +//////////////////////////////////////////////////////////////// +bool likeOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + if (type == TSDB_DATA_TYPE_BINARY) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH; + } else if (type == TSDB_DATA_TYPE_NCHAR) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + return WCSPatternMatch((wchar_t*)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; + } else { return false; } - - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) == 0; -} - -//////////////////////////////////////////////////////////////// -bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH; -} - -bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - return WCSPatternMatch((wchar_t*)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; } //////////////////////////////////////////////////////////////// @@ -221,360 +198,237 @@ bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { * During pre-filter stage, if there is one element that locates in [minval, maxval], * the filter function will return true. */ -bool nequal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int8_t *)minval == *(int8_t *)maxval) { - return (*(int8_t *)minval != pFilter->filterInfo.lowerBndi); - } - - return true; -} - -bool nequal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int16_t *)minval == *(int16_t *)maxval) { - return (*(int16_t *)minval != pFilter->filterInfo.lowerBndi); +// TODO not equal need to refactor +bool notEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { + int64_t minv = -1, maxv = -1; + GET_TYPED_DATA(minv, int64_t, type, minval); + GET_TYPED_DATA(maxv, int64_t, type, maxval); + + if (minv == maxv) { + return minv != pFilterInfo->lowerBndi; + } + return true; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + uint64_t minv = 0, maxv = 0; + GET_TYPED_DATA(minv, uint64_t, type, minval); + GET_TYPED_DATA(maxv, uint64_t, type, maxval); + + if (minv == maxv) { + return minv != pFilterInfo->lowerBndi; + } + return true; + } else if (IS_FLOAT_TYPE(type)) { + double minv = -1, maxv = -1; + GET_TYPED_DATA(minv, double, type, minval); + GET_TYPED_DATA(maxv, double, type, maxval); + + if (minv == maxv) { + return !FLT_EQUAL(minv, pFilterInfo->lowerBndd); + } + return true; + } else if (type == TSDB_DATA_TYPE_BINARY) { + if (pFilterInfo->len != varDataLen(minval)) { + return true; + } + return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) != 0; + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pFilterInfo->len != varDataLen(minval)) { + return true; + } + return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; + } else { + return false; } - - return true; } -bool nequal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int32_t *)minval == *(int32_t *)maxval) { - return (*(int32_t *)minval != pFilter->filterInfo.lowerBndi); - } - +//////////////////////////////////////////////////////////////// +// dummy filter, not used +bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { return true; } -bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(int64_t *)minval == *(int64_t *)maxval) { - return (*(int64_t *)minval != pFilter->filterInfo.lowerBndi); - } - +bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { return true; } -bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(float *)minval == *(float *)maxval) { - return (*(float *)minval != pFilter->filterInfo.lowerBndd); +/////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_ii(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UTINYINT: + return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_SMALLINT: + return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_USMALLINT: + return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_INT: + return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UINT: + return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UBIGINT: + return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_FLOAT: + return FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd) && + FLT_GREATEREQUAL(*(float *)maxval, pFilterInfo->lowerBndd); + case TSDB_DATA_TYPE_DOUBLE: + return (*(double *)minval <= pFilterInfo->upperBndd && *(double *)maxval >= pFilterInfo->lowerBndd); + default: + return false; } - - return true; } -bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (*(double *)minval == *(double *)maxval) { - return (*(double *)minval != pFilter->filterInfo.lowerBndd); +bool rangeFilter_ee(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UTINYINT: + return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_SMALLINT: + return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_USMALLINT: + return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_INT: + return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UINT: + return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UBIGINT: + return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_FLOAT: + return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval > pFilterInfo->lowerBndd)); + case TSDB_DATA_TYPE_DOUBLE: + return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd)); + default: + return false; } - - return true; } -bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (pFilter->filterInfo.len != varDataLen(minval)) { - return true; +bool rangeFilter_ie(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UTINYINT: + return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_SMALLINT: + return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_USMALLINT: + return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_INT: + return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UINT: + return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UBIGINT: + return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_FLOAT: + return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval >= pFilterInfo->lowerBndd)); + case TSDB_DATA_TYPE_DOUBLE: + return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval >= pFilterInfo->lowerBndd)); + default: + return false; } - - return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) != 0; } -bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { - if (pFilter->filterInfo.len > pFilter->bytes) { - return true; +bool rangeFilter_ei(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { + SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UTINYINT: + return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_SMALLINT: + return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_USMALLINT: + return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_INT: + return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UINT: + return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_UBIGINT: + return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi)); + case TSDB_DATA_TYPE_FLOAT: + return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd) && + FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd); + case TSDB_DATA_TYPE_DOUBLE: + return ((*(double *)minval <= pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd)); + default: + return false; } - - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; -} -//////////////////////////////////////////////////////////////// -bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) { - return true; -} - -bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) { - return true; -} - -//////////////////////////////////////////////////////////////// - -bool rangeFilter_i32_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i32_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minvalfilterInfo.upperBndi &&*(int32_t *)maxval> pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i32_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval < pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i32_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -/////////////////////////////////////////////////////////////////////////////// -bool rangeFilter_i8_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i8_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minvalfilterInfo.upperBndi &&*(int8_t *)maxval> pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i8_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval < pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i8_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -///////////////////////////////////////////////////////////////////////////////////// -bool rangeFilter_i16_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i16_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minvalfilterInfo.upperBndi &&*(int16_t *)maxval> pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i16_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval < pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i16_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -//////////////////////////////////////////////////////////////////////// -bool rangeFilter_i64_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i64_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minvalfilterInfo.upperBndi &&*(int64_t *)maxval> pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i64_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval < pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); -} - -bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval > pFilter->filterInfo.lowerBndi); -} - -//////////////////////////////////////////////////////////////////////// -bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); -} - -bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minvalfilterInfo.upperBndd &&*(float *)maxval> pFilter->filterInfo.lowerBndd); -} - -bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval < pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); -} - -bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd); -} - -////////////////////////////////////////////////////////////////////////// -bool rangeFilter_dd_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); -} - -bool rangeFilter_dd_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)minvalfilterInfo.upperBndd &&*(double *)maxval> pFilter->filterInfo.lowerBndd); -} - -bool rangeFilter_dd_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)minval < pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); -} - -bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval > pFilter->filterInfo.lowerBndd); } //////////////////////////////////////////////////////////////////////////// -bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - less_i8, - large_i8, - equal_i8, - lessEqual_i8, - largeEqual_i8, - nequal_i8, - NULL, - isNull_filter, - notNull_filter, -}; - -bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - less_i16, - large_i16, - equal_i16, - lessEqual_i16, - largeEqual_i16, - nequal_i16, - NULL, - isNull_filter, - notNull_filter, -}; - -bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - less_i32, - large_i32, - equal_i32, - lessEqual_i32, - largeEqual_i32, - nequal_i32, - NULL, - isNull_filter, - notNull_filter, -}; - -bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - less_i64, - large_i64, - equal_i64, - lessEqual_i64, - largeEqual_i64, - nequal_i64, - NULL, - isNull_filter, - notNull_filter, +bool (*filterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = { + NULL, + lessOperator, + greaterOperator, + equalOperator, + lessEqualOperator, + greaterEqualOperator, + notEqualOperator, + likeOperator, + isNullOperator, + notNullOperator, }; -bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - less_ds, - large_ds, - equal_ds, - lessEqual_ds, - largeEqual_ds, - nequal_ds, - NULL, - isNull_filter, - notNull_filter, +bool (*rangeFilterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = { + NULL, + rangeFilter_ee, + rangeFilter_ie, + rangeFilter_ei, + rangeFilter_ii, }; -bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - less_dd, - large_dd, - equal_dd, - lessEqual_dd, - largeEqual_dd, - nequal_dd, - NULL, - isNull_filter, - notNull_filter, -}; - -bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { - NULL, - NULL, - NULL, - equal_str, - NULL, - NULL, - nequal_str, - like_str, - isNull_filter, - notNull_filter, -}; - -bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { - NULL, - NULL, - NULL, - equal_nchar, - NULL, - NULL, - nequal_nchar, - like_nchar, - isNull_filter, - notNull_filter, -}; - -bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - rangeFilter_i8_ee, - rangeFilter_i8_ie, - rangeFilter_i8_ei, - rangeFilter_i8_ii, -}; - -bool (*rangeFilterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - rangeFilter_i16_ee, - rangeFilter_i16_ie, - rangeFilter_i16_ei, - rangeFilter_i16_ii, -}; - -bool (*rangeFilterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - rangeFilter_i32_ee, - rangeFilter_i32_ie, - rangeFilter_i32_ei, - rangeFilter_i32_ii, -}; - -bool (*rangeFilterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - rangeFilter_i64_ee, - rangeFilter_i64_ie, - rangeFilter_i64_ei, - rangeFilter_i64_ii, -}; - -bool (*rangeFilterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - rangeFilter_ds_ee, - rangeFilter_ds_ie, - rangeFilter_ds_ei, - rangeFilter_ds_ii, -}; - -bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { - NULL, - rangeFilter_dd_ee, - rangeFilter_dd_ie, - rangeFilter_dd_ei, - rangeFilter_dd_ii, -}; - -__filter_func_t* getRangeFilterFuncArray(int32_t type) { - switch(type) { - case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8; - case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8; - case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16; - case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32; - case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter - case TSDB_DATA_TYPE_BIGINT: return rangeFilterFunc_i64; - case TSDB_DATA_TYPE_FLOAT: return rangeFilterFunc_ds; - case TSDB_DATA_TYPE_DOUBLE: return rangeFilterFunc_dd; - default:return NULL; +__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr) { + __filter_func_t funcFp = NULL; + + if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) && + (upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) { + if (lowerOptr == TSDB_RELATION_GREATER_EQUAL) { + if (upperOptr == TSDB_RELATION_LESS_EQUAL) { + funcFp = rangeFilterOperators[4]; + } else { + funcFp = rangeFilterOperators[2]; + } + } else { + if (upperOptr == TSDB_RELATION_LESS_EQUAL) { + funcFp = rangeFilterOperators[3]; + } else { + funcFp = rangeFilterOperators[1]; + } + } + } else { // set callback filter function + if (lowerOptr != TSDB_RELATION_INVALID) { + funcFp = filterOperators[lowerOptr]; + + // invalid filter condition: %d", pQInfo, type + if (upperOptr != TSDB_RELATION_INVALID) { + return NULL; + } + } else { + funcFp = filterOperators[upperOptr]; + } } -} -__filter_func_t* getValueFilterFuncArray(int32_t type) { - switch(type) { - case TSDB_DATA_TYPE_BOOL: return filterFunc_i8; - case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8; - case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16; - case TSDB_DATA_TYPE_INT: return filterFunc_i32; - case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter - case TSDB_DATA_TYPE_BIGINT: return filterFunc_i64; - case TSDB_DATA_TYPE_FLOAT: return filterFunc_ds; - case TSDB_DATA_TYPE_DOUBLE: return filterFunc_dd; - case TSDB_DATA_TYPE_BINARY: return filterFunc_str; - case TSDB_DATA_TYPE_NCHAR: return filterFunc_nchar; - default: return NULL; - } + return funcFp; } diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 35e5906d1f797c2f53f3906bf93c6797ac766c3e..ae25a752347edc4f7c53aade845716d637f9edd7 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -143,6 +143,9 @@ SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { SHistogramInfo* pHisto = (SHistogramInfo*)pBuf; pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo)); + for(int32_t i = 0; i < numOfBins; ++i) { + pHisto->elems[i].val = -DBL_MAX; + } pHisto->maxEntries = numOfBins; @@ -158,8 +161,8 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } #if defined(USE_ARRAYLIST) - int32_t idx = vnodeHistobinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries); + int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); + assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); if ((*pHisto)->elems[idx].val == val && idx >= 0) { (*pHisto)->elems[idx].num += 1; @@ -171,17 +174,17 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { assert((*pHisto)->elems[idx - 1].val <= val); + } else { + assert((*pHisto)->elems[idx].val > val); } - - assert((*pHisto)->elems[idx].val > val); - } else { - assert((*pHisto)->elems[(*pHisto)->numOfEntries].val < val); + } else if ((*pHisto)->numOfElems > 0) { + assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val); } histogramCreateBin(*pHisto, idx, val); } #else - tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); SHistBin* entry = calloc(1, sizeof(SHistBin)); entry->val = val; @@ -214,7 +217,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } tSkipListKey kx = - tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); pLast = tSkipListGetOne((*pHisto)->pList, &kx); } } else { @@ -356,7 +359,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { return 0; } -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val) { +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { int32_t end = len - 1; int32_t start = 0; @@ -466,7 +469,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) { */ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { #if defined(USE_ARRAYLIST) - int32_t slotIdx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, v); + int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v); if (pHisto->elems[slotIdx].val != v) { slotIdx -= 1; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 7d71d9f7f15ad88d0bc1565047ae7686320d2832..5937fdb68f0fb7d8915b427cacd627f96bc02d1c 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -54,10 +54,19 @@ SSqlInfo qSQLParse(const char *pStr) { case TK_QUESTION: case TK_ILLEGAL: { - snprintf(sqlInfo.pzErrMsg, tListLen(sqlInfo.pzErrMsg), "unrecognized token: \"%s\"", t0.z); + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unrecognized token: \"%s\"", t0.z); sqlInfo.valid = false; goto abort_parse; } + + case TK_HEX: + case TK_OCT: + case TK_BIN:{ + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + default: Parse(pParser, t0.type, t0, &sqlInfo); if (sqlInfo.valid == false) { @@ -71,13 +80,13 @@ abort_parse: return sqlInfo; } -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { +tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pDistinct, SStrToken *pToken) { if (pList == NULL) { pList = calloc(1, sizeof(tSQLExprList)); } if (pList->nAlloc <= pList->nExpr) { - pList->nAlloc = (pList->nAlloc << 1) + 4; + pList->nAlloc = (pList->nAlloc << 1u) + 4; pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); if (pList->a == 0) { pList->nExpr = pList->nAlloc = 0; @@ -87,7 +96,7 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken assert(pList->a != 0); if (pNode || pToken) { - struct tSQLExprItem *pItem = &pList->a[pList->nExpr++]; + struct tSqlExprItem *pItem = &pList->a[pList->nExpr++]; memset(pItem, 0, sizeof(*pItem)); pItem->pNode = pNode; if (pToken) { // set the as clause @@ -97,66 +106,67 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken strdequote(pItem->aliasName); } + pItem->distinct = (pDistinct != NULL); } return pList; } -void tSQLExprListDestroy(tSQLExprList *pList) { +void tSqlExprListDestroy(tSQLExprList *pList) { if (pList == NULL) return; for (int32_t i = 0; i < pList->nExpr; ++i) { if (pList->a[i].aliasName != NULL) { free(pList->a[i].aliasName); } - tSQLExprDestroy(pList->a[i].pNode); + tSqlExprDestroy(pList->a[i].pNode); } free(pList->a); free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { - tSQLExpr *pSQLExpr = calloc(1, sizeof(tSQLExpr)); +tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType) { + tSQLExpr *pSqlExpr = calloc(1, sizeof(tSQLExpr)); if (pToken != NULL) { - pSQLExpr->token = *pToken; + pSqlExpr->token = *pToken; } if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { toTSDBType(pToken->type); - tVariantCreate(&pSQLExpr->val, pToken); - pSQLExpr->nSQLOptr = optrType; + tVariantCreate(&pSqlExpr->val, pToken); + pSqlExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // use microsecond by default - pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSqlExpr->val.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSQLExpr->val.i64Key); + int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->val.i64); if (ret != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSQLExpr->nSQLOptr = TK_TIMESTAMP; + pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->nSQLOptr = TK_TIMESTAMP; } else { // it must be the column name (tk_id) if it is not the number assert(optrType == TK_ID || optrType == TK_ALL); if (pToken != NULL) { - pSQLExpr->colInfo = *pToken; + pSqlExpr->colInfo = *pToken; } - pSQLExpr->nSQLOptr = optrType; + pSqlExpr->nSQLOptr = optrType; } - return pSQLExpr; + return pSqlExpr; } /* * pList is the parameters for function with id(optType) * function name is denoted by pFunctionToken */ -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { +tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { if (pFuncToken == NULL) return NULL; tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); @@ -177,7 +187,7 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt * create binary expression in this procedure * if the expr is arithmetic, calculate the result and set it to tSQLExpr Object */ -tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { +tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { @@ -200,39 +210,39 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { switch (optrType) { case TK_PLUS: { - pExpr->val.i64Key = pLeft->val.i64Key + pRight->val.i64Key; + pExpr->val.i64 = pLeft->val.i64 + pRight->val.i64; break; } case TK_MINUS: { - pExpr->val.i64Key = pLeft->val.i64Key - pRight->val.i64Key; + pExpr->val.i64 = pLeft->val.i64 - pRight->val.i64; break; } case TK_STAR: { - pExpr->val.i64Key = pLeft->val.i64Key * pRight->val.i64Key; + pExpr->val.i64 = pLeft->val.i64 * pRight->val.i64; break; } case TK_DIVIDE: { pExpr->nSQLOptr = TK_FLOAT; pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->val.dKey = (double)pLeft->val.i64Key / pRight->val.i64Key; + pExpr->val.dKey = (double)pLeft->val.i64 / pRight->val.i64; break; } case TK_REM: { - pExpr->val.i64Key = pLeft->val.i64Key % pRight->val.i64Key; + pExpr->val.i64 = pLeft->val.i64 % pRight->val.i64; break; } } - tSQLExprDestroy(pLeft); - tSQLExprDestroy(pRight); + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); } else if ((pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_INTEGER) || (pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_FLOAT) || (pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_FLOAT)) { pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE; pExpr->nSQLOptr = TK_FLOAT; - double left = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64Key; - double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64Key; + double left = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64; + double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64; switch (optrType) { case TK_PLUS: { @@ -257,8 +267,8 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { } } - tSQLExprDestroy(pLeft); - tSQLExprDestroy(pRight); + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); } else { pExpr->nSQLOptr = optrType; @@ -278,7 +288,7 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { pExpr->nSQLOptr = optrType; pExpr->pLeft = pLeft; - if (pRight == NULL) { + if (pLeft != NULL && pRight == NULL) { pRight = calloc(1, sizeof(tSQLExpr)); } @@ -288,7 +298,29 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { return pExpr; } -void tSQLExprNodeDestroy(tSQLExpr *pExpr) { + + +tSQLExpr *tSqlExprClone(tSQLExpr *pSrc) { + tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); + + memcpy(pExpr, pSrc, sizeof(*pSrc)); + + if (pSrc->pLeft) { + pExpr->pLeft = tSqlExprClone(pSrc->pLeft); + } + + if (pSrc->pRight) { + pExpr->pRight = tSqlExprClone(pSrc->pRight); + } + + //we don't clone pParam now because clone is only used for between/and + assert(pSrc->pParam == NULL); + + return pExpr; +} + + +void tSqlExprNodeDestroy(tSQLExpr *pExpr) { if (pExpr == NULL) { return; } @@ -297,20 +329,21 @@ void tSQLExprNodeDestroy(tSQLExpr *pExpr) { tVariantDestroy(&pExpr->val); } - tSQLExprListDestroy(pExpr->pParam); + tSqlExprListDestroy(pExpr->pParam); free(pExpr); } -void tSQLExprDestroy(tSQLExpr *pExpr) { +void tSqlExprDestroy(tSQLExpr *pExpr) { if (pExpr == NULL) { return; } - tSQLExprDestroy(pExpr->pLeft); - tSQLExprDestroy(pExpr->pRight); - - tSQLExprNodeDestroy(pExpr); + tSqlExprDestroy(pExpr->pLeft); + pExpr->pLeft = NULL; + tSqlExprDestroy(pExpr->pRight); + pExpr->pRight = NULL; + tSqlExprNodeDestroy(pExpr); } SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) { @@ -366,13 +399,13 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int return pList; } -void setDBName(SStrToken *pCpxName, SStrToken *pDB) { - pCpxName->type = pDB->type; - pCpxName->z = pDB->z; - pCpxName->n = pDB->n; +void setDbName(SStrToken *pCpxName, SStrToken *pDb) { + pCpxName->type = pDb->type; + pCpxName->z = pDb->z; + pCpxName->n = pDb->n; } -void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { +void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); // truncate the column name @@ -384,61 +417,128 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) pField->name[pName->n] = 0; pField->type = pType->type; - pField->bytes = pType->bytes; + if(!isValidDataType(pField->type)){ + pField->bytes = 0; + } else { + pField->bytes = pType->bytes; + } } -void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { - pField->type = -1; +static int32_t tryParseNameTwoParts(SStrToken *type) { + int32_t t = -1; + + char* str = strndup(type->z, type->n); + if (str == NULL) { + return t; + } + + char* p = strtok(str, " "); + if (p == NULL) { + tfree(str); + return t; + } else { + char* unsign = strtok(NULL, " "); + if (unsign == NULL) { + tfree(str); + return t; + } - for (int8_t i = 0; i < tListLen(tDataTypeDesc); ++i) { - if ((strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0) && - (type->n == tDataTypeDesc[i].nameLen)) { - pField->type = i; - pField->bytes = tDataTypeDesc[i].nSize; - - if (i == TSDB_DATA_TYPE_NCHAR) { - /* - * for nchar, the TOKENTYPE is the number of character, so the length is the - * number of bytes in UCS-4 format, which is 4 times larger than the - * number of characters - */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - // we have to postpone reporting the error because it cannot be done here - // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow - bytes = TSDB_MAX_NCHAR_LEN + 1; - } else { - bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } - pField->bytes = (int16_t)bytes; + if (strncasecmp(unsign, "UNSIGNED", 8) == 0) { + for(int32_t j = TSDB_DATA_TYPE_TINYINT; j <= TSDB_DATA_TYPE_BIGINT; ++j) { + if (strcasecmp(p, tDataTypes[j].name) == 0) { + t = j; + break; } - } else if (i == TSDB_DATA_TYPE_BINARY) { - /* for binary, the TOKENTYPE is the length of binary */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - // refer comment for NCHAR above - bytes = TSDB_MAX_BINARY_LEN + 1; - } else { - bytes += VARSTR_HEADER_SIZE; - } - pField->bytes = (int16_t)bytes; - } } + + tfree(str); + + if (t == -1) { + return -1; + } + + switch(t) { + case TSDB_DATA_TYPE_TINYINT: return TSDB_DATA_TYPE_UTINYINT; + case TSDB_DATA_TYPE_SMALLINT: return TSDB_DATA_TYPE_USMALLINT; + case TSDB_DATA_TYPE_INT: return TSDB_DATA_TYPE_UINT; + case TSDB_DATA_TYPE_BIGINT: return TSDB_DATA_TYPE_UBIGINT; + default: + return -1; + } + + } else { + tfree(str); + return -1; + } + } +} + +void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type) { + // set the field type invalid + pField->type = -1; + pField->name[0] = 0; + + int32_t i = 0; + while (i < tListLen(tDataTypes)) { + if ((type->n == tDataTypes[i].nameLen) && + (strncasecmp(type->z, tDataTypes[i].name, tDataTypes[i].nameLen) == 0)) { break; } + + i += 1; + } + + // no qualified data type found, try unsigned data type + if (i == tListLen(tDataTypes)) { + i = tryParseNameTwoParts(type); + if (i == -1) { + return; + } + } + + pField->type = i; + pField->bytes = tDataTypes[i].bytes; + + if (i == TSDB_DATA_TYPE_NCHAR) { + /* + * for nchar, the TOKENTYPE is the number of character, so the length is the + * number of bytes in UCS-4 format, which is 4 times larger than the number of characters + */ + if (type->type == 0) { + pField->bytes = 0; + } else { + int32_t bytes = -(int32_t)(type->type); + if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + // we have to postpone reporting the error because it cannot be done here + // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow + bytes = TSDB_MAX_NCHAR_LEN + 1; + } else { + bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } + pField->bytes = (int16_t)bytes; + } + } else if (i == TSDB_DATA_TYPE_BINARY) { + /* for binary, the TOKENTYPE is the length of binary */ + if (type->type == 0) { + pField->bytes = 0; + } else { + int32_t bytes = -(int32_t)(type->type); + if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + // refer comment for NCHAR above + bytes = TSDB_MAX_BINARY_LEN + 1; + } else { + bytes += VARSTR_HEADER_SIZE; + } + + pField->bytes = (int16_t)bytes; + } } } /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, +SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); @@ -474,21 +574,29 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, return pQuery; } -void freeVariant(void *pItem) { +static void freeVariant(void *pItem) { tVariantListItem* p = (tVariantListItem*) pItem; tVariantDestroy(&p->pVar); } +void freeCreateTableInfo(void* p) { + SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p; + taosArrayDestroy(pInfo->pTagNames); + taosArrayDestroyEx(pInfo->pTagVals, freeVariant); + tfree(pInfo->fullname); + tfree(pInfo->tagdata.data); +} + void doDestroyQuerySql(SQuerySQL *pQuerySql) { if (pQuerySql == NULL) { return; } - - tSQLExprListDestroy(pQuerySql->pSelection); + + tSqlExprListDestroy(pQuerySql->pSelection); pQuerySql->pSelection = NULL; - - tSQLExprDestroy(pQuerySql->pWhere); + + tSqlExprDestroy(pQuerySql->pWhere); pQuerySql->pWhere = NULL; taosArrayDestroyEx(pQuerySql->pSortOrder, freeVariant); @@ -519,31 +627,30 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { tfree(pClause->pClause); } -SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pStableName, - SArray *pTagVals, SQuerySQL *pSelect, int32_t type) { +SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type) { SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); switch (type) { case TSQL_CREATE_TABLE: { pCreate->colInfo.pColumns = pCols; - assert(pTagVals == NULL && pTags == NULL); + assert(pTags == NULL); break; } case TSQL_CREATE_STABLE: { pCreate->colInfo.pColumns = pCols; pCreate->colInfo.pTagColumns = pTags; - assert(pTagVals == NULL && pTags != NULL && pCols != NULL); - break; - } - case TSQL_CREATE_TABLE_FROM_STABLE: { - pCreate->usingInfo.pTagVals = pTagVals; - pCreate->usingInfo.stableName = *pStableName; + assert(pTags != NULL && pCols != NULL); break; } case TSQL_CREATE_STREAM: { pCreate->pSelect = pSelect; break; } + + case TSQL_CREATE_TABLE_FROM_STABLE: { + assert(0); + } + default: assert(false); } @@ -552,11 +659,25 @@ SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pSt return pCreate; } -SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type) { - SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); +SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists) { + SCreatedTableInfo info; + memset(&info, 0, sizeof(SCreatedTableInfo)); + + info.name = *pToken; + info.pTagNames = pTagNames; + info.pTagVals = pTagVals; + info.stableName = *pTableName; + info.igExist = (igExists->n > 0)? 1:0; + + return info; +} + +SAlterTableInfo *tAlterTableSqlElems(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableType) { + SAlterTableInfo *pAlterTable = calloc(1, sizeof(SAlterTableInfo)); - pAlterTable->name = *pMeterName; + pAlterTable->name = *pTableName; pAlterTable->type = type; + pAlterTable->tableType = tableType; if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { pAlterTable->pAddColumns = pCols; @@ -573,35 +694,40 @@ SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray return pAlterTable; } -void SQLInfoDestroy(SSqlInfo *pInfo) { +void* destroyCreateTableSql(SCreateTableSQL* pCreate) { + doDestroyQuerySql(pCreate->pSelect); + + taosArrayDestroy(pCreate->colInfo.pColumns); + taosArrayDestroy(pCreate->colInfo.pTagColumns); + + taosArrayDestroyEx(pCreate->childTableInfo, freeCreateTableInfo); + tfree(pCreate); + + return NULL; +} + +void SqlInfoDestroy(SSqlInfo *pInfo) { if (pInfo == NULL) return; if (pInfo->type == TSDB_SQL_SELECT) { destroyAllSelectClause(&pInfo->subclauseInfo); } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) { - SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo; - doDestroyQuerySql(pCreateTableInfo->pSelect); - - taosArrayDestroy(pCreateTableInfo->colInfo.pColumns); - taosArrayDestroy(pCreateTableInfo->colInfo.pTagColumns); - - taosArrayDestroyEx(pCreateTableInfo->usingInfo.pTagVals, freeVariant); - tfree(pInfo->pCreateTableInfo); + pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant); taosArrayDestroy(pInfo->pAlterInfo->pAddColumns); - + tfree(pInfo->pAlterInfo->tagData.data); tfree(pInfo->pAlterInfo); } else { - if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) { - free(pInfo->pDCLInfo->a); + if (pInfo->pMiscInfo != NULL) { + taosArrayDestroy(pInfo->pMiscInfo->a); } - if (pInfo->pDCLInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) { - taosArrayDestroyEx(pInfo->pDCLInfo->dbOpt.keep, freeVariant); + if (pInfo->pMiscInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) { + taosArrayDestroyEx(pInfo->pMiscInfo->dbOpt.keep, freeVariant); } - tfree(pInfo->pDCLInfo); + tfree(pInfo->pMiscInfo); } } @@ -624,7 +750,7 @@ SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { return pSubclause; } -SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type) { +SSqlInfo*setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type) { pInfo->type = type; if (type == TSDB_SQL_SELECT) { @@ -634,8 +760,8 @@ SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, pInfo->pCreateTableInfo = pSqlExprInfo; } - if (pMeterName != NULL) { - pInfo->pCreateTableInfo->name = *pMeterName; + if (pTableName != NULL) { + pInfo->pCreateTableInfo->name = *pTableName; } return pInfo; @@ -653,62 +779,54 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists) { - pInfo->pCreateTableInfo->name = *pMeterName; +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists) { + pInfo->pCreateTableInfo->name = *pTableNameToken; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } -void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { - if (pTokenList->nAlloc <= pTokenList->nTokens) { // - pTokenList->nAlloc = (pTokenList->nAlloc << 1) + 4; - pTokenList->a = realloc(pTokenList->a, pTokenList->nAlloc * sizeof(pTokenList->a[0])); - if (pTokenList->a == 0) { - pTokenList->nTokens = pTokenList->nAlloc = 0; - } - } -} - -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken) { - if (pToken == NULL) return NULL; - - if (pTokenList == NULL) pTokenList = calloc(1, sizeof(tDCLSQL)); - - tTokenListBuyMoreSpace(pTokenList); - pTokenList->a[pTokenList->nTokens++] = *pToken; - - return pTokenList; -} - void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { pInfo->type = type; + if (nParam == 0) { + return; + } - if (nParam == 0) return; - if (pInfo->pDCLInfo == NULL) pInfo->pDCLInfo = (tDCLSQL *)calloc(1, sizeof(tDCLSQL)); + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); + pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); + } va_list va; va_start(va, nParam); - while (nParam-- > 0) { + while ((nParam--) > 0) { SStrToken *pToken = va_arg(va, SStrToken *); - pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); + taosArrayPush(pInfo->pMiscInfo->a, pToken); } va_end(va); } -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t tableType) { pInfo->type = type; - pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); - pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); + + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); + pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); + } + + taosArrayPush(pInfo->pMiscInfo->a, pToken); + + pInfo->pMiscInfo->existsCheck = (existsCheck->n == 1); + pInfo->pMiscInfo->tableType = tableType; } void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); } pInfo->type = TSDB_SQL_SHOW; - SShowInfo* pShowInfo = &pInfo->pDCLInfo->showOpt; + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; pShowInfo->showType = type; if (prefix != NULL && prefix->type != 0) { @@ -724,54 +842,54 @@ void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* } } -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists) { +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists) { pInfo->type = type; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); } - pInfo->pDCLInfo->dbOpt = *pDB; - pInfo->pDCLInfo->dbOpt.dbname = *pToken; - pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} + pInfo->pMiscInfo->dbOpt = *pDB; + pInfo->pMiscInfo->dbOpt.dbname = *pToken; + pInfo->pMiscInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo) { pInfo->type = type; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); } - pInfo->pDCLInfo->acctOpt = *pAcctInfo; + pInfo->pMiscInfo->acctOpt = *pAcctInfo; assert(pName != NULL); - pInfo->pDCLInfo->user.user = *pName; + pInfo->pMiscInfo->user.user = *pName; if (pPwd != NULL) { - pInfo->pDCLInfo->user.passwd = *pPwd; + pInfo->pMiscInfo->user.passwd = *pPwd; } } -void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { +void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->type = TSDB_SQL_CREATE_USER; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); } assert(pName != NULL && pPasswd != NULL); - pInfo->pDCLInfo->user.user = *pName; - pInfo->pDCLInfo->user.passwd = *pPasswd; + pInfo->pMiscInfo->user.user = *pName; + pInfo->pMiscInfo->user.passwd = *pPasswd; } -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { pInfo->type = TSDB_SQL_ALTER_USER; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); } assert(pName != NULL); - SUserInfo* pUser = &pInfo->pDCLInfo->user; + SUserInfo* pUser = &pInfo->pMiscInfo->user; pUser->type = type; pUser->user = *pName; @@ -788,18 +906,17 @@ void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* } } -void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { +void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *id) { pInfo->type = type; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); } - assert(ip != NULL); - - pInfo->pDCLInfo->ip = *ip; + assert(id != NULL); + pInfo->pMiscInfo->id = *id; } -void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { pDBInfo->compressionLevel = -1; pDBInfo->walLevel = -1; @@ -818,5 +935,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->keep = NULL; pDBInfo->update = -1; + pDBInfo->cachelast = 0; memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 3bdc0d477f777faa71a8e5d2264021932dddeea4..523fa42547e34e9f0bb9ac23868ab3dc67eeec34 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -12,14 +12,14 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "os.h" #include "qPercentile.h" #include "qResultbuf.h" -#include "os.h" #include "queryLog.h" #include "taosdef.h" -#include "tulog.h" #include "tcompare.h" +#include "ttype.h" #define DEFAULT_NUM_OF_SLOT 1024 @@ -48,25 +48,15 @@ static tFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) } static void resetBoundingBox(MinMaxEntry* range, int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_BIGINT: { - range->i64MaxVal = INT64_MIN; - range->i64MinVal = INT64_MAX; - break; - }; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: { - range->iMaxVal = INT32_MIN; - range->iMinVal = INT32_MAX; - break; - }; - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_FLOAT: { - range->dMaxVal = -DBL_MAX; - range->dMinVal = DBL_MAX; - break; - } + if (IS_SIGNED_NUMERIC_TYPE(type)) { + range->i64MaxVal = INT64_MIN; + range->i64MinVal = INT64_MAX; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + range->u64MaxVal = 0; + range->u64MinVal = UINT64_MAX; + } else { + range->dMaxVal = -DBL_MAX; + range->dMinVal = DBL_MAX; } } @@ -75,23 +65,15 @@ static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, d return -1; } - switch(type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - range->iMinVal = (int32_t) minval; - range->iMaxVal = (int32_t) maxval; - break; - - case TSDB_DATA_TYPE_BIGINT: - range->i64MinVal = (int64_t) minval; - range->i64MaxVal = (int64_t) maxval; - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - range->dMinVal = minval; - range->dMaxVal = maxval; - break; + if (IS_SIGNED_NUMERIC_TYPE(type)) { + range->i64MinVal = (int64_t) minval; + range->i64MaxVal = (int64_t) maxval; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)){ + range->u64MinVal = (uint64_t) minval; + range->u64MaxVal = (uint64_t) maxval; + } else { + range->dMinVal = minval; + range->dMaxVal = maxval; } return 0; @@ -120,179 +102,106 @@ double findOnlyResult(tMemBucket *pMemBucket) { tFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); assert(pPage->num == 1); - switch (pMemBucket->type) { - case TSDB_DATA_TYPE_INT: - return *(int32_t *)pPage->data; - case TSDB_DATA_TYPE_SMALLINT: - return *(int16_t *)pPage->data; - case TSDB_DATA_TYPE_TINYINT: - return *(int8_t *)pPage->data; - case TSDB_DATA_TYPE_BIGINT: - return (double)(*(int64_t *)pPage->data); - case TSDB_DATA_TYPE_DOUBLE: { - double dv = GET_DOUBLE_VAL(pPage->data); - return dv; - } - case TSDB_DATA_TYPE_FLOAT: { - float fv = GET_FLOAT_VAL(pPage->data); - return fv; - } - default: - return 0; - } + double v = 0; + GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data); + return v; } return 0; } -int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) { - int64_t v = *(int64_t *)value; - int32_t index = -1; - - int32_t halfSlot = pBucket->numOfSlots >> 1; -// int32_t bits = 32;//bitsOfNumber(pBucket->numOfSlots) - 1; +int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pBucket->type, value); - if (pBucket->range.i64MaxVal == INT64_MIN) { - if (v >= 0) { - index = (v >> (64 - 9)) + halfSlot; - } else { // v<0 - index = ((-v) >> (64 - 9)); - index = -index + (halfSlot - 1); - } + int32_t index = -1; + if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) { return index; + } + + // divide the value range into 1024 buckets + uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; + if (span < pBucket->numOfSlots) { + int64_t delta = v - pBucket->range.i64MinVal; + index = (delta % pBucket->numOfSlots); } else { - // out of range - if (v < pBucket->range.i64MinVal || v > pBucket->range.i64MaxVal) { - return -1; + double slotSpan = (double)span / pBucket->numOfSlots; + index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan); + if (v == pBucket->range.i64MaxVal) { + index -= 1; } - - // todo hash for bigint and float and double - int64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; - if (span < pBucket->numOfSlots) { - int32_t delta = (int32_t)(v - pBucket->range.i64MinVal); - index = delta % pBucket->numOfSlots; - } else { - double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan); - if (v == pBucket->range.i64MaxVal) { - index -= 1; - } - } - - return index; } + + assert(index >= 0 && index < pBucket->numOfSlots); + return index; } -// todo refactor to more generic -int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { - int32_t v = 0; - switch(pBucket->type) { - case TSDB_DATA_TYPE_SMALLINT: v = *(int16_t*) value; break; - case TSDB_DATA_TYPE_TINYINT: v = *(int8_t*) value; break; - default: v = *(int32_t*) value;break; - } +int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { + int64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pBucket->type, value); int32_t index = -1; - if (pBucket->range.iMaxVal == INT32_MIN) { - /* - * taking negative integer into consideration, - * there is only half of pBucket->segs available for non-negative integer - */ - int32_t halfSlot = pBucket->numOfSlots >> 1; - int32_t bits = 32;//bitsOfNumber(pBucket->numOfSlots) - 1; - - if (v >= 0) { - index = (v >> (bits - 9)) + halfSlot; - } else { // v < 0 - index = ((-v) >> (32 - 9)); - index = -index + (halfSlot - 1); - } + if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) { return index; + } + + // divide the value range into 1024 buckets + uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal; + if (span < pBucket->numOfSlots) { + int64_t delta = v - pBucket->range.u64MinVal; + index = (int32_t) (delta % pBucket->numOfSlots); } else { - // out of range - if (v < pBucket->range.iMinVal || v > pBucket->range.iMaxVal) { - return -1; - } - - // divide a range of [iMinVal, iMaxVal] into 1024 buckets - int32_t span = pBucket->range.iMaxVal - pBucket->range.iMinVal; - if (span < pBucket->numOfSlots) { - int32_t delta = v - pBucket->range.iMinVal; - index = (delta % pBucket->numOfSlots); - } else { - double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.iMinVal) / slotSpan); - if (v == pBucket->range.iMaxVal) { - index -= 1; - } + double slotSpan = (double)span / pBucket->numOfSlots; + index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan); + if (v == pBucket->range.u64MaxVal) { + index -= 1; } - - return index; } + + assert(index >= 0 && index < pBucket->numOfSlots); + return index; } int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { - double v = GET_DOUBLE_VAL(value); - int32_t index = -1; - - if (pBucket->range.dMinVal == DBL_MAX) { - /* - * taking negative integer into consideration, - * there is only half of pBucket->segs available for non-negative integer - */ - double x = DBL_MAX / (pBucket->numOfSlots >> 1); - double posx = (v + DBL_MAX) / x; - return ((int32_t)posx) % pBucket->numOfSlots; + double v = 0; + if (pBucket->type == TSDB_DATA_TYPE_FLOAT) { + v = GET_FLOAT_VAL(value); } else { + v = GET_DOUBLE_VAL(value); + } - // out of range - if (v < pBucket->range.dMinVal || v > pBucket->range.dMaxVal) { - return -1; - } + int32_t index = -1; - // divide a range of [dMinVal, dMaxVal] into 1024 buckets - double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; - if (span < pBucket->numOfSlots) { - int32_t delta = (int32_t)(v - pBucket->range.dMinVal); - index = (delta % pBucket->numOfSlots); - } else { - double slotSpan = span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); - if (v == pBucket->range.dMaxVal) { - index -= 1; - } - } + if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { + return index; + } - if (index < 0 || index > pBucket->numOfSlots) { - uError("error in hash process. slot id: %d", index); + // divide a range of [dMinVal, dMaxVal] into 1024 buckets + double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; + if (span < pBucket->numOfSlots) { + int32_t delta = (int32_t)(v - pBucket->range.dMinVal); + index = (delta % pBucket->numOfSlots); + } else { + double slotSpan = span / pBucket->numOfSlots; + index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); + if (v == pBucket->range.dMaxVal) { + index -= 1; } - - return index; } + + assert(index >= 0 && index < pBucket->numOfSlots); + return index; } static __perc_hash_func_t getHashFunc(int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: { - return tBucketIntHash; - }; - - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_FLOAT: { - return tBucketDoubleHash; - }; - - case TSDB_DATA_TYPE_BIGINT: { - return tBucketBigIntHash; - }; - - default: { - return NULL; - } + if (IS_SIGNED_NUMERIC_TYPE(type)) { + return tBucketIntHash; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + return tBucketUintHash; + } else { + return tBucketDoubleHash; } } @@ -322,7 +231,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, pBucket->maxCapacity = 200000; if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { - uError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); + qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); free(pBucket); return NULL; } @@ -332,7 +241,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, pBucket->hashFunc = getHashFunc(pBucket->type); if (pBucket->hashFunc == NULL) { - uError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); + qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); free(pBucket); return NULL; } @@ -351,7 +260,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, return NULL; } - uDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); + qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); return pBucket; } @@ -366,77 +275,41 @@ void tMemBucketDestroy(tMemBucket *pBucket) { } void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { - switch (dataType) { - case TSDB_DATA_TYPE_INT: { - int32_t val = *(int32_t *)data; - if (r->iMinVal > val) { - r->iMinVal = val; - } + if (IS_SIGNED_NUMERIC_TYPE(dataType)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, dataType, data); - if (r->iMaxVal < val) { - r->iMaxVal = val; - } - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - int64_t val = *(int64_t *)data; - if (r->i64MinVal > val) { - r->i64MinVal = val; - } - - if (r->i64MaxVal < val) { - r->i64MaxVal = val; - } - break; - }; - case TSDB_DATA_TYPE_SMALLINT: { - int32_t val = *(int16_t *)data; - if (r->iMinVal > val) { - r->iMinVal = val; - } - - if (r->iMaxVal < val) { - r->iMaxVal = val; - } - break; - }; - case TSDB_DATA_TYPE_TINYINT: { - int32_t val = *(int8_t *)data; - if (r->iMinVal > val) { - r->iMinVal = val; - } + if (r->i64MinVal > v) { + r->i64MinVal = v; + } - if (r->iMaxVal < val) { - r->iMaxVal = val; - } + if (r->i64MaxVal < v) { + r->i64MaxVal = v; + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, dataType, data); - break; - }; - case TSDB_DATA_TYPE_DOUBLE: { - // double val = *(double *)data; - double val = GET_DOUBLE_VAL(data); - if (r->dMinVal > val) { - r->dMinVal = val; - } + if (r->i64MinVal > v) { + r->i64MinVal = v; + } - if (r->dMaxVal < val) { - r->dMaxVal = val; - } - break; - }; - case TSDB_DATA_TYPE_FLOAT: { - double val = GET_FLOAT_VAL(data); + if (r->i64MaxVal < v) { + r->i64MaxVal = v; + } + } else if (IS_FLOAT_TYPE(dataType)) { + double v = 0; + GET_TYPED_DATA(v, double, dataType, data); - if (r->dMinVal > val) { - r->dMinVal = val; - } + if (r->dMinVal > v) { + r->dMinVal = v; + } - if (r->dMaxVal < val) { - r->dMaxVal = val; - } - break; - }; - default: { assert(false); } + if (r->dMaxVal < v) { + r->dMaxVal = v; + } + } else { + assert(0); } } @@ -446,17 +319,18 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { assert(pBucket != NULL && data != NULL && size > 0); - pBucket->total += (int32_t)size; - + int32_t count = 0; int32_t bytes = pBucket->bytes; for (int32_t i = 0; i < size; ++i) { char *d = (char *) data + i * bytes; int32_t index = (pBucket->hashFunc)(pBucket, d); - if (index == -1) { // the value is out of range, do not add it into bucket - return -1; + if (index < 0) { + continue; } + count += 1; + tMemBucketSlot *pSlot = &pBucket->pSlots[index]; tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); @@ -483,64 +357,11 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.size += 1; } + pBucket->total += count; return 0; } //////////////////////////////////////////////////////////////////////////////////////////// -static UNUSED_FUNC void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) { - *minVal = DBL_MAX; - *maxVal = -DBL_MAX; - - for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { - tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; - if (pSlot->info.size == 0) { - continue; - } - - switch (pMemBucket->type) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: { - double minv = pSlot->range.iMinVal; - double maxv = pSlot->range.iMaxVal; - - if (*minVal > minv) { - *minVal = minv; - } - if (*maxVal < maxv) { - *maxVal = maxv; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_FLOAT: { - double minv = pSlot->range.dMinVal; - double maxv = pSlot->range.dMaxVal; - - if (*minVal > minv) { - *minVal = minv; - } - if (*maxVal < maxv) { - *maxVal = maxv; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - double minv = (double)pSlot->range.i64MinVal; - double maxv = (double)pSlot->range.i64MaxVal; - - if (*minVal > minv) { - *minVal = minv; - } - if (*maxVal < maxv) { - *maxVal = maxv; - } - break; - } - } - } -} - /* * * now, we need to find the minimum value of the next slot for @@ -559,7 +380,6 @@ static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int3 } static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); -char *getFirstElemOfMemBuffer(tMemBucketSlot *pSeg, int32_t slotIdx, tFilePage *pPage); static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) { assert(isIdenticalData(pMemBucket, slotIndex)); @@ -567,24 +387,12 @@ static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) { tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex]; double finalResult = 0.0; - switch (pMemBucket->type) { - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_INT: { - finalResult = pSlot->range.iMinVal; - break; - } - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - finalResult = pSlot->range.dMinVal; - break; - }; - - case TSDB_DATA_TYPE_BIGINT: { - finalResult = (double)pSlot->range.i64MinVal; - break; - } + if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { + finalResult = (double) pSlot->range.i64MinVal; + } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { + finalResult = (double) pSlot->range.u64MinVal; + } else { + finalResult = (double) pSlot->range.dMinVal; } return finalResult; @@ -610,26 +418,16 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) double maxOfThisSlot = 0; double minOfNextSlot = 0; - switch (pMemBucket->type) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: { - maxOfThisSlot = pSlot->range.iMaxVal; - minOfNextSlot = next.iMinVal; - break; - }; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - maxOfThisSlot = pSlot->range.dMaxVal; - minOfNextSlot = next.dMinVal; - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - maxOfThisSlot = (double)pSlot->range.i64MaxVal; - minOfNextSlot = (double)next.i64MinVal; - break; - } - }; + if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { + maxOfThisSlot = (double) pSlot->range.i64MaxVal; + minOfNextSlot = (double) next.i64MinVal; + } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { + maxOfThisSlot = (double) pSlot->range.u64MaxVal; + minOfNextSlot = (double) next.u64MinVal; + } else { + maxOfThisSlot = (double) pSlot->range.dMaxVal; + minOfNextSlot = (double) next.dMinVal; + } assert(minOfNextSlot > maxOfThisSlot); @@ -646,38 +444,8 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) char *nextVal = thisVal + pMemBucket->bytes; double td = 1.0, nd = 1.0; - switch (pMemBucket->type) { - case TSDB_DATA_TYPE_SMALLINT: { - td = *(int16_t *)thisVal; - nd = *(int16_t *)nextVal; - break; - } - case TSDB_DATA_TYPE_TINYINT: { - td = *(int8_t *)thisVal; - nd = *(int8_t *)nextVal; - break; - } - case TSDB_DATA_TYPE_INT: { - td = *(int32_t *)thisVal; - nd = *(int32_t *)nextVal; - break; - }; - case TSDB_DATA_TYPE_FLOAT: { - td = GET_FLOAT_VAL(thisVal); - nd = GET_FLOAT_VAL(nextVal); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - td = GET_DOUBLE_VAL(thisVal); - nd = GET_DOUBLE_VAL(nextVal); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - td = (double)*(int64_t *)thisVal; - nd = (double)*(int64_t *)nextVal; - break; - } - } + GET_TYPED_DATA(td, double, pMemBucket->type, thisVal); + GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); double val = (1 - fraction) * td + fraction * nd; tfree(buffer); @@ -690,7 +458,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) // try next round pMemBucket->times += 1; - uDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times); + qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times); pMemBucket->range = pSlot->range; pMemBucket->total = 0; @@ -735,20 +503,14 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) { MinMaxEntry* pRange = &pMemBucket->range; - switch(pMemBucket->type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - return fabs(percent - 100) < DBL_EPSILON? pRange->iMaxVal:pRange->iMinVal; - case TSDB_DATA_TYPE_BIGINT: { - double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); - return v; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal; - default: - return -1; + if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { + double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); + return v; + } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { + double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal); + return v; + } else { + return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal; } } @@ -765,40 +527,9 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) { tMemBucketSlot *pSeg = &pMemBucket->pSlots[index]; - if (pMemBucket->type == TSDB_DATA_TYPE_INT || pMemBucket->type == TSDB_DATA_TYPE_BIGINT || - pMemBucket->type == TSDB_DATA_TYPE_SMALLINT || pMemBucket->type == TSDB_DATA_TYPE_TINYINT) { - return pSeg->range.i64MinVal == pSeg->range.i64MaxVal; - } - - if (pMemBucket->type == TSDB_DATA_TYPE_FLOAT || pMemBucket->type == TSDB_DATA_TYPE_DOUBLE) { + if (IS_FLOAT_TYPE(pMemBucket->type)) { return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON; + } else { + return pSeg->range.i64MinVal == pSeg->range.i64MaxVal; } - - return false; -} - -/* - * get the first element of one slot into memory. - * if no data of current slot in memory, load it from disk - */ -char *getFirstElemOfMemBuffer(tMemBucketSlot *pSeg, int32_t slotIdx, tFilePage *pPage) { -// STSBuf *pMemBuffer = pSeg->pBuffer[slotIdx]; - char *thisVal = NULL; - -// if (pSeg->pBuffer[slotIdx]->numOfTotal != 0) { -//// thisVal = pSeg->pBuffer[slotIdx]->pHead->item.data; -// } else { -// /* -// * no data in memory, load one page into memory -// */ -// tFlushoutInfo *pFlushInfo = &pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[0]; -// assert(pFlushInfo->numOfPages == pMemBuffer->fileMeta.nFileSize); -// int32_t ret; -// ret = fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET); -// UNUSED(ret); -// size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file); -// UNUSED(sz); -// thisVal = pPage->data; -// } - return thisVal; } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index bc7243830d836287cde808076cfb24de80fa574d..d45e76c2fd07f37afa0ffad0f9db65d5e1948c43 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -313,7 +313,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 // allocate buf if (availablePage == NULL) { - pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES); + pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES + 2); // add extract bytes in case of zipped buffer increased. } else { pi->pData = availablePage; } diff --git a/src/query/src/qSyntaxtreefunction.c b/src/query/src/qSyntaxtreefunction.c deleted file mode 100644 index 7f7fca2c1e0ca6ce85f0df1c1b6e6dce82d8c000..0000000000000000000000000000000000000000 --- a/src/query/src/qSyntaxtreefunction.c +++ /dev/null @@ -1,1256 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "qSyntaxtreefunction.h" -#include "taosdef.h" -#include "tutil.h" - -#define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ - { \ - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; \ - \ - if ((len1) == (len2)) { \ - for (; i < (len2) && i >= 0; i += step, (out) += 1) { \ - if (isNull((char *)&((left)[i]), _left_type) || isNull((char *)&((right)[i]), _right_type)) { \ - setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(left)[i] op(right)[i]; \ - } \ - } else if ((len1) == 1) { \ - for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ - if (isNull((char *)(left), _left_type) || isNull((char *)&(right)[i], _right_type)) { \ - setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(left)[0] op(right)[i]; \ - } \ - } else if ((len2) == 1) { \ - for (; i >= 0 && i < (len1); i += step, (out) += 1) { \ - if (isNull((char *)&(left)[i], _left_type) || isNull((char *)(right), _right_type)) { \ - setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(left)[i] op(right)[0]; \ - } \ - } \ - } - -#define ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ - { \ - int32_t i = (_ord == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ - int32_t step = (_ord == TSDB_ORDER_ASC) ? 1 : -1; \ - \ - if (len1 == (len2)) { \ - for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ - if (isNull((char *)&(left[i]), _left_type) || isNull((char *)&(right[i]), _right_type)) { \ - setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[i])) * (right)[i]; \ - } \ - } else if (len1 == 1) { \ - for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ - if (isNull((char *)(left), _left_type) || isNull((char *)&((right)[i]), _right_type)) { \ - setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(left)[0] - ((int64_t)(((double)(left)[0]) / (right)[i])) * (right)[i]; \ - } \ - } else if ((len2) == 1) { \ - for (; i >= 0 && i < len1; i += step, (out) += 1) { \ - if (isNull((char *)&((left)[i]), _left_type) || isNull((char *)(right), _right_type)) { \ - setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[0])) * (right)[0]; \ - } \ - } \ - } - -#define ARRAY_LIST_ADD(left, right, _left_type, _right_type, len1, len2, out, _ord) \ - ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, +, TSDB_DATA_TYPE_DOUBLE, _ord) -#define ARRAY_LIST_SUB(left, right, _left_type, _right_type, len1, len2, out, _ord) \ - ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, -, TSDB_DATA_TYPE_DOUBLE, _ord) -#define ARRAY_LIST_MULTI(left, right, _left_type, _right_type, len1, len2, out, _ord) \ - ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, *, TSDB_DATA_TYPE_DOUBLE, _ord) -#define ARRAY_LIST_DIV(left, right, _left_type, _right_type, len1, len2, out, _ord) \ - ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, /, TSDB_DATA_TYPE_DOUBLE, _ord) -#define ARRAY_LIST_REM(left, right, _left_type, _right_type, len1, len2, out, _ord) \ - ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, %, TSDB_DATA_TYPE_DOUBLE, _ord) - -#define TYPE_CONVERT_DOUBLE_RES(left, right, out, _type_left, _type_right, _type_res) \ - _type_left * pLeft = (_type_left *)(left); \ - _type_right *pRight = (_type_right *)(right); \ - _type_res * pOutput = (_type_res *)(out); - -#define TYPE_CONVERT(left, right, out, _type_left, _type_right) \ - TYPE_CONVERT_DOUBLE_RES(left, right, out, _type_left, _type_right, double) - -void calc_fn_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] + pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] + pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] + pRight[0]; - } - } -} - -void calc_fn_i32_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t); - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_i8_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i8_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_i16_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i16_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_i16_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i16_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_i64_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i64_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i16_i64_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i64_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_i64_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i64_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i16_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i64_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i16_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i64_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_f_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -///////////////////////////////////////////////////////////////////////////////////////////////////////// -void calc_fn_i32_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] - pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[0] - pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] - pRight[0]; - } - } -} - -void calc_fn_i32_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -///////////////////////////////////////////////////////////////////////////////////////////////////////// -void calc_fn_i32_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] * pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] * pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] * pRight[0]; - } - } -} - -void calc_fn_i32_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_i8_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i8_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_i16_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i16_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_i16_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i16_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_i64_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i64_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i16_i64_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i64_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_i64_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_i64_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i16_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i64_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_f_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i8_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i16_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i32_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_i64_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_fn_f_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_fn_d_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////// -void calc_fn_i32_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] / pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] / pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] / pRight[0]; - } - } -} - -void calc_fn_i32_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////// - -void calc_fn_i32_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[i])) * pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] - ((int64_t)(((double)pLeft[0]) / pRight[i])) * pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[0])) * pRight[0]; - } - } -} - -void calc_fn_i32_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i32_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - double * pRight = (double *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[i])) * pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] - ((int64_t)(((double)pLeft[0]) / pRight[i])) * pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[0])) * pRight[0]; - } - } -} - -void calc_fn_i8_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i8_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i16_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_i64_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_f_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_fn_d_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -/* - * the following are two-dimensional array list of callback function . - */ -_bi_consumer_fn_t add_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_add, calc_fn_i8_i16_add, calc_fn_i8_i32_add, calc_fn_i8_i64_add, calc_fn_i8_f_add, calc_fn_i8_d_add, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_add, calc_fn_i16_i16_add, calc_fn_i16_i32_add, calc_fn_i16_i64_add, calc_fn_i16_f_add, calc_fn_i16_d_add, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_add, calc_fn_i32_i16_add, calc_fn_i32_i32_add, calc_fn_i32_i64_add, calc_fn_i32_f_add, calc_fn_i32_d_add, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_add, calc_fn_i64_i16_add, calc_fn_i64_i32_add, calc_fn_i64_i64_add, calc_fn_i64_f_add, calc_fn_i64_d_add, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_add, calc_fn_f_i16_add, calc_fn_f_i32_add, calc_fn_f_i64_add, calc_fn_f_f_add, calc_fn_f_d_add, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_add, calc_fn_d_i16_add, calc_fn_d_i32_add, calc_fn_d_i64_add, calc_fn_d_f_add, calc_fn_d_d_add, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -}; - -_bi_consumer_fn_t sub_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_sub, calc_fn_i8_i16_sub, calc_fn_i8_i32_sub, calc_fn_i8_i64_sub, calc_fn_i8_f_sub, calc_fn_i8_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_sub, calc_fn_i16_i16_sub, calc_fn_i16_i32_sub, calc_fn_i16_i64_sub, calc_fn_i16_f_sub, calc_fn_i16_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_sub, calc_fn_i32_i16_sub, calc_fn_i32_i32_sub, calc_fn_i32_i64_sub, calc_fn_i32_f_sub, calc_fn_i32_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_sub, calc_fn_i64_i16_sub, calc_fn_i64_i32_sub, calc_fn_i64_i64_sub, calc_fn_i64_f_sub, calc_fn_i64_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_sub, calc_fn_f_i16_sub, calc_fn_f_i32_sub, calc_fn_f_i64_sub, calc_fn_f_f_sub, calc_fn_f_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_sub, calc_fn_d_i16_sub, calc_fn_d_i32_sub, calc_fn_d_i64_sub, calc_fn_d_f_sub, calc_fn_d_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -}; - -_bi_consumer_fn_t multi_function_arraylist[][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_multi, calc_fn_i8_i16_multi, calc_fn_i8_i32_multi, calc_fn_i8_i64_multi, calc_fn_i8_f_multi, calc_fn_i8_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_multi, calc_fn_i16_i16_multi, calc_fn_i16_i32_multi, calc_fn_i16_i64_multi, calc_fn_i16_f_multi, calc_fn_i16_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_multi, calc_fn_i32_i16_multi, calc_fn_i32_i32_multi, calc_fn_i32_i64_multi, calc_fn_i32_f_multi, calc_fn_i32_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_multi, calc_fn_i64_i16_multi, calc_fn_i64_i32_multi, calc_fn_i64_i64_multi, calc_fn_i64_f_multi, calc_fn_i64_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_multi, calc_fn_f_i16_multi, calc_fn_f_i32_multi, calc_fn_f_i64_multi, calc_fn_f_f_multi, calc_fn_f_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_multi, calc_fn_d_i16_multi, calc_fn_d_i32_multi, calc_fn_d_i64_multi, calc_fn_d_f_multi, calc_fn_d_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -}; - -_bi_consumer_fn_t div_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_div, calc_fn_i8_i16_div, calc_fn_i8_i32_div, calc_fn_i8_i64_div, calc_fn_i8_f_div, calc_fn_i8_d_div, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_div, calc_fn_i16_i16_div, calc_fn_i16_i32_div, calc_fn_i16_i64_div, calc_fn_i16_f_div, calc_fn_i16_d_div, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_div, calc_fn_i32_i16_div, calc_fn_i32_i32_div, calc_fn_i32_i64_div, calc_fn_i32_f_div, calc_fn_i32_d_div, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_div, calc_fn_i64_i16_div, calc_fn_i64_i32_div, calc_fn_i64_i64_div, calc_fn_i64_f_div, calc_fn_i64_d_div, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_div, calc_fn_f_i16_div, calc_fn_f_i32_div, calc_fn_f_i64_div, calc_fn_f_f_div, calc_fn_f_d_div, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_div, calc_fn_d_i16_div, calc_fn_d_i32_div, calc_fn_d_i64_div, calc_fn_d_f_div, calc_fn_d_d_div, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -}; - -_bi_consumer_fn_t rem_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, - {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_rem, calc_fn_i8_i16_rem, calc_fn_i8_i32_rem, calc_fn_i8_i64_rem, calc_fn_i8_f_rem, calc_fn_i8_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_rem, calc_fn_i16_i16_rem, calc_fn_i16_i32_rem, calc_fn_i16_i64_rem, calc_fn_i16_f_rem, calc_fn_i16_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_rem, calc_fn_i32_i16_rem, calc_fn_i32_i32_rem, calc_fn_i32_i64_rem, calc_fn_i32_f_rem, calc_fn_i32_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_rem, calc_fn_i64_i16_rem, calc_fn_i64_i32_rem, calc_fn_i64_i64_rem, calc_fn_i64_f_rem, calc_fn_i64_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_rem, calc_fn_f_i16_rem, calc_fn_f_i32_rem, calc_fn_f_i64_rem, calc_fn_f_f_rem, calc_fn_f_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_rem, calc_fn_d_i16_rem, calc_fn_d_i32_rem, calc_fn_d_i64_rem, calc_fn_d_f_rem, calc_fn_d_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////// - -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr) { - switch (optr) { - case TSDB_BINARY_OP_ADD: - return add_function_arraylist[leftType][rightType]; - case TSDB_BINARY_OP_SUBTRACT: - return sub_function_arraylist[leftType][rightType]; - case TSDB_BINARY_OP_MULTIPLY: - return multi_function_arraylist[leftType][rightType]; - case TSDB_BINARY_OP_DIVIDE: - return div_function_arraylist[leftType][rightType]; - case TSDB_BINARY_OP_REMAINDER: - return rem_function_arraylist[leftType][rightType]; - default: - assert(0); - return NULL; - } - - assert(0); - return NULL; -} diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 98545c8ef3930587b6f7bd2c5b8d40ded0066d52..b55f813b5b1225d0b8e05b87e11b0d19ed2ce200 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -129,6 +129,7 @@ static SKeyword keywordTable[] = { {"PRECISION", TK_PRECISION}, {"LP", TK_LP}, {"RP", TK_RP}, + {"UNSIGNED", TK_UNSIGNED}, {"TAGS", TK_TAGS}, {"USING", TK_USING}, {"AS", TK_AS}, @@ -238,6 +239,8 @@ static SKeyword keywordTable[] = { {"SUM_IRATE", TK_SUM_IRATE}, {"AVG_RATE", TK_AVG_RATE}, {"AVG_IRATE", TK_AVG_IRATE}, + {"CACHELAST", TK_CACHELAST}, + {"DISTINCT", TK_DISTINCT}, }; static const char isIdChar[] = { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index d0c59fe5efe08aa1c9dbe3bbe65db139b074ebee..1f43c5b33cca044744a2783a251f76bcb827d34a 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -19,13 +19,19 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { if (pTSBuf == NULL) { return NULL; } + + pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath("join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "w+"); + pTSBuf->f = fopen(pTSBuf->path, "wb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; } + + if (!autoDelete) { + unlink(pTSBuf->path); + } if (NULL == allocResForTSBuf(pTSBuf)) { return NULL; @@ -37,8 +43,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { tsBufResetPos(pTSBuf); pTSBuf->cur.order = TSDB_ORDER_ASC; - - pTSBuf->autoDelete = autoDelete; + pTSBuf->tsOrder = order; return pTSBuf; @@ -49,10 +54,12 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { if (pTSBuf == NULL) { return NULL; } + + pTSBuf->autoDelete = autoDelete; tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "r+"); + pTSBuf->f = fopen(pTSBuf->path, "rb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; @@ -129,7 +136,6 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // ascending by default pTSBuf->cur.order = TSDB_ORDER_ASC; - pTSBuf->autoDelete = autoDelete; // tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); @@ -147,8 +153,10 @@ void* tsBufDestroy(STSBuf* pTSBuf) { tfree(pTSBuf->pData); tfree(pTSBuf->block.payload); - - fclose(pTSBuf->f); + + if (!pTSBuf->remainOpen) { + fclose(pTSBuf->f); + } if (pTSBuf->autoDelete) { // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); @@ -260,7 +268,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.i64, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); } else { trueLen = 0; metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); @@ -343,7 +351,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); UNUSED(sz); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = fread(&pBlock->tag.i64, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); UNUSED(sz); } @@ -947,7 +955,7 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64Key, elem.ts); + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64, elem.ts); } } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 6c845b012faade8fbb693d3406013b8264b0c4a7..b7a7fd28e9dc2f013457e22446ff7bc41d3fef6c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -19,18 +19,15 @@ #include "qExecutor.h" #include "qUtil.h" +#include "tbuffer.h" +#include "tlosertree.h" +#include "queryLog.h" -static int32_t getResultRowKeyInfo(SResultRow* pResult, int16_t type, char** key, int16_t* bytes) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - *key = varDataVal(pResult->key); - *bytes = varDataLen(pResult->key); - } else { - *key = (char*) &pResult->win.skey; - *bytes = tDataTypeDesc[type].nSize; - } - - return 0; -} +typedef struct SCompSupporter { + STableQueryInfo **pTableQueryInfo; + int32_t *rowIndex; + int32_t order; +} SCompSupporter; int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; @@ -43,7 +40,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { +int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { pResultRowInfo->capacity = size; pResultRowInfo->type = type; @@ -59,10 +56,11 @@ int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t return TSDB_CODE_SUCCESS; } -void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) { +void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { if (pResultRowInfo == NULL) { return; } + if (pResultRowInfo->capacity == 0) { assert(pResultRowInfo->pResult == NULL); return; @@ -77,7 +75,7 @@ void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) { tfree(pResultRowInfo->pResult); } -void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { +void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) { return; } @@ -95,81 +93,10 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultR pResultRowInfo->curIndex = -1; pResultRowInfo->size = 0; - - pResultRowInfo->startTime = TSKEY_INITIAL_VAL; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } -void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { - SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; - if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) { - return; - } - - int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); - assert(num >= 0 && num <= numOfClosed); - - int16_t type = pResultRowInfo->type; - int64_t uid = getResultInfoUId(pRuntimeEnv); - - char *key = NULL; - int16_t bytes = -1; - - for (int32_t i = 0; i < num; ++i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { // remove the window slot from hash table - getResultRowKeyInfo(pResult, type, &key, &bytes); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - } else { - break; - } - } - - int32_t remain = pResultRowInfo->size - num; - - // clear all the closed windows from the window list - for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type); - } - - // move the unclosed window in the front of the window list - for (int32_t k = remain; k < pResultRowInfo->size; ++k) { - SResultRow *pWindowRes = pResultRowInfo->pResult[k]; - clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); - } - - pResultRowInfo->size = remain; - - for (int32_t k = 0; k < pResultRowInfo->size; ++k) { - SResultRow *pResult = pResultRowInfo->pResult[k]; - getResultRowKeyInfo(pResult, type, &key, &bytes); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - - int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - assert(p != NULL); - - int32_t v = (*p - num); - assert(v >= 0 && v <= pResultRowInfo->size); - - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); - } - - pResultRowInfo->curIndex = -1; -} - -void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { - SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; - if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) { - return; - } - - int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); - clearFirstNWindowRes(pRuntimeEnv, numOfClosed); -} - -int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) { +int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { int32_t i = 0; while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { ++i; @@ -178,73 +105,40 @@ int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) { return i; } -void closeAllTimeWindow(SResultRowInfo *pResultRowInfo) { +void closeAllResultRows(SResultRowInfo *pResultRowInfo) { assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - if (pResultRowInfo->pResult[i]->closed) { + SResultRow* pRow = pResultRowInfo->pResult[i]; + if (pRow->closed) { continue; } - pResultRowInfo->pResult[i]->closed = true; + pRow->closed = true; } } -/* - * remove the results that are not the FIRST time window that spreads beyond the - * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time. - * NOTE: remove redundant, only when the result set order equals to traverse order - */ -void removeRedundantWindow(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) { - assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); - if (pResultRowInfo->size <= 1) { - return; - } - - // get the result order - int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1; - if (order != resultOrder) { - return; - } - - int32_t i = 0; - if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey; - while (i < pResultRowInfo->size && (ekey < lastKey)) { - ++i; - } - } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) { - ++i; - } - } - - if (i < pResultRowInfo->size) { - pResultRowInfo->size = (i + 1); - } -} - -bool isWindowResClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { +bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { return (getResultRow(pResultRowInfo, slot)->closed == true); } -void closeTimeWindow(SResultRowInfo *pResultRowInfo, int32_t slot) { +void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { getResultRow(pResultRowInfo, slot)->closed = true; } -void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) { - if (pWindowRes == NULL) { +void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { + if (pResultRow == NULL) { return; } // the result does not put into the SDiskbasedResultBuf, ignore it. - if (pWindowRes->pageId >= 0) { - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); + if (pResultRow->pageId >= 0) { + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { - SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i]; + SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i]; - char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); + char * s = getPosInResultPage(pRuntimeEnv, i, pResultRow, page); size_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes; memset(s, 0, size); @@ -252,56 +146,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16 } } - pWindowRes->numOfRows = 0; - pWindowRes->pageId = -1; - pWindowRes->rowId = -1; - pWindowRes->closed = false; - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tfree(pWindowRes->key); - } else { - pWindowRes->win = TSWINDOW_INITIALIZER; - } -} - -/** - * The source window result pos attribution of the source window result does not assign to the destination, - * since the attribute of "Pos" is bound to each window result when the window result is created in the - * disk-based result buffer. - */ -void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src, int16_t type) { - dst->numOfRows = src->numOfRows; + pResultRow->numOfRows = 0; + pResultRow->pageId = -1; + pResultRow->rowId = -1; + pResultRow->closed = false; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - dst->key = realloc(dst->key, varDataTLen(src->key)); - varDataCopy(dst->key, src->key); + tfree(pResultRow->key); } else { - dst->win = src->win; - } - dst->closed = src->closed; - - int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; - - for (int32_t i = 0; i < nOutputCols; ++i) { - SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i); - SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i); - -// char *buf = pDst->interResultBuf; - memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes); -// pDst->interResultBuf = buf; // restore the allocated buffer - - // copy the result info struct -// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes); - - // copy the output buffer data from src to dst, the position info keep unchanged - tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId); - char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); - - tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId); - char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage); - size_t s = pRuntimeEnv->pQuery->pExpr1[i].bytes; - - memcpy(dstBuf, srcBuf, s); + pResultRow->win = TSWINDOW_INITIALIZER; } } @@ -310,7 +163,7 @@ SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRo return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); } -size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) { +size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); } @@ -386,16 +239,334 @@ void* destroyResultRowPool(SResultRowPool* p) { return NULL; } -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { - if (!pRuntimeEnv->stableQuery) { - return 0; // for simple table query, the uid is always set to be 0; +void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen) { + uint32_t numOfGroup = (uint32_t) taosArrayGetSize(pRes); + tbufWriteUint32(bw, numOfGroup); + tbufWriteUint16(bw, tagLen); + + for(int32_t i = 0; i < numOfGroup; ++i) { + SInterResult* pOne = taosArrayGet(pRes, i); + if (tagLen > 0) { + tbufWriteBinary(bw, pOne->tags, tagLen); + } + + uint32_t numOfCols = (uint32_t) taosArrayGetSize(pOne->pResult); + tbufWriteUint32(bw, numOfCols); + for(int32_t j = 0; j < numOfCols; ++j) { + SStddevInterResult* p = taosArrayGet(pOne->pResult, j); + uint32_t numOfRows = (uint32_t) taosArrayGetSize(p->pResult); + + tbufWriteUint16(bw, p->colId); + tbufWriteUint32(bw, numOfRows); + + for(int32_t k = 0; k < numOfRows; ++k) { + SResPair v = *(SResPair*) taosArrayGet(p->pResult, k); + tbufWriteDouble(bw, v.avg); + tbufWriteInt64(bw, v.key); + } + } + } +} + +SArray* interResFromBinary(const char* data, int32_t len) { + SBufferReader br = tbufInitReader(data, len, false); + uint32_t numOfGroup = tbufReadUint32(&br); + uint16_t tagLen = tbufReadUint16(&br); + + char* tag = NULL; + if (tagLen > 0) { + tag = calloc(1, tagLen); + } + + SArray* pResult = taosArrayInit(4, sizeof(SInterResult)); + + for(int32_t i = 0; i < numOfGroup; ++i) { + if (tagLen > 0) { + memset(tag, 0, tagLen); + tbufReadToBinary(&br, tag, tagLen); + } + + uint32_t numOfCols = tbufReadUint32(&br); + + SArray* p = taosArrayInit(numOfCols, sizeof(SStddevInterResult)); + for(int32_t j = 0; j < numOfCols; ++j) { + int16_t colId = tbufReadUint16(&br); + int32_t numOfRows = tbufReadUint32(&br); + + SStddevInterResult interRes = {.colId = colId, .pResult = taosArrayInit(4, sizeof(struct SResPair)),}; + for(int32_t k = 0; k < numOfRows; ++k) { + SResPair px = {0}; + px.avg = tbufReadDouble(&br); + px.key = tbufReadInt64(&br); + + taosArrayPush(interRes.pResult, &px); + } + + taosArrayPush(p, &interRes); + } + + char* p1 = NULL; + if (tagLen > 0) { + p1 = malloc(tagLen); + memcpy(p1, tag, tagLen); + } + + SInterResult d = {.pResult = p, .tags = p1,}; + taosArrayPush(pResult, &d); + } + + tfree(tag); + return pResult; +} + +void freeInterResult(void* param) { + SInterResult* pResult = (SInterResult*) param; + tfree(pResult->tags); + + int32_t numOfCols = (int32_t) taosArrayGetSize(pResult->pResult); + for(int32_t i = 0; i < numOfCols; ++i) { + SStddevInterResult *p = taosArrayGet(pResult->pResult, i); + taosArrayDestroy(p->pResult); } + taosArrayDestroy(pResult->pResult); +} + +void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { + assert(pGroupResInfo != NULL); + + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; + pGroupResInfo->index = 0; +} + +void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo, int32_t offset) { + if (pGroupResInfo->pRows != NULL) { + taosArrayDestroy(pGroupResInfo->pRows); + } + + pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES); + pGroupResInfo->index = offset; + + assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); +} + +bool hasRemainData(SGroupResInfo* pGroupResInfo) { + if (pGroupResInfo->pRows == NULL) { + return false; + } + + return pGroupResInfo->index < taosArrayGetSize(pGroupResInfo->pRows); +} + +bool incNextGroup(SGroupResInfo* pGroupResInfo) { + return (++pGroupResInfo->currentGroup) < pGroupResInfo->totalGroup; +} + +int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { + assert(pGroupResInfo != NULL); + if (pGroupResInfo->pRows == 0) { + return 0; + } + + return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); +} + +static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { SQuery* pQuery = pRuntimeEnv->pQuery; - if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) { + + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functionId = pQuery->pExpr1[j].base.functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + continue; + } + + SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j); + assert(pResultInfo != NULL); + + if (pResultInfo->numOfRes > 0) { + return pResultInfo->numOfRes; + } + } + + return 0; +} + +static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { + int32_t left = *(int32_t *)pLeft; + int32_t right = *(int32_t *)pRight; + + SCompSupporter * supporter = (SCompSupporter *)param; + + int32_t leftPos = supporter->rowIndex[left]; + int32_t rightPos = supporter->rowIndex[right]; + + /* left source is exhausted */ + if (leftPos == -1) { + return 1; + } + + /* right source is exhausted*/ + if (rightPos == -1) { + return -1; + } + + STableQueryInfo** pList = supporter->pTableQueryInfo; + + SResultRowInfo *pWindowResInfo1 = &(pList[left]->resInfo); + SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); + TSKEY leftTimestamp = pWindowRes1->win.skey; + + SResultRowInfo *pWindowResInfo2 = &(pList[right]->resInfo); + SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); + TSKEY rightTimestamp = pWindowRes2->win.skey; + + if (leftTimestamp == rightTimestamp) { return 0; } - STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); - return id->uid; -} \ No newline at end of file + if (supporter->order == TSDB_ORDER_ASC) { + return (leftTimestamp > rightTimestamp)? 1:-1; + } else { + return (leftTimestamp < rightTimestamp)? 1:-1; + } +} + +static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, void* qinfo) { + bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery); + + int32_t code = TSDB_CODE_SUCCESS; + + int32_t *posList = NULL; + SLoserTreeInfo *pTree = NULL; + STableQueryInfo **pTableQueryInfoList = NULL; + + size_t size = taosArrayGetSize(pTableList); + if (pGroupResInfo->pRows == NULL) { + pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); + } + + posList = calloc(size, sizeof(int32_t)); + pTableQueryInfoList = malloc(POINTER_BYTES * size); + + if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { + qError("QInfo:%p failed alloc memory", qinfo); + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; + } + + int32_t numOfTables = 0; + for (int32_t i = 0; i < size; ++i) { + STableQueryInfo *item = taosArrayGetP(pTableList, i); + if (item->resInfo.size > 0) { + pTableQueryInfoList[numOfTables++] = item; + } + } + + // there is no data in current group + // no need to merge results since only one table in each group + if (numOfTables == 0) { + goto _end; + } + + SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQuery->order.order}; + + int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); + if (ret != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; + } + + int64_t lastTimestamp = ascQuery? INT64_MIN:INT64_MAX; + int64_t startt = taosGetTimestampMs(); + + while (1) { + int32_t tableIndex = pTree->pNode[0].index; + + SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo; + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); + + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); + if (num <= 0) { + cs.rowIndex[tableIndex] += 1; + + if (cs.rowIndex[tableIndex] >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; + if (--numOfTables == 0) { // all input sources are exhausted + break; + } + } + } else { + assert((pWindowRes->win.skey >= lastTimestamp && ascQuery) || (pWindowRes->win.skey <= lastTimestamp && !ascQuery)); + + if (pWindowRes->win.skey != lastTimestamp) { + taosArrayPush(pGroupResInfo->pRows, &pWindowRes); + pWindowRes->numOfRows = (uint32_t) num; + } + + lastTimestamp = pWindowRes->win.skey; + + // move to the next row of current entry + if ((++cs.rowIndex[tableIndex]) >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; + + // all input sources are exhausted + if ((--numOfTables) == 0) { + break; + } + } + } + + tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries); + } + + int64_t endt = taosGetTimestampMs(); + + qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", qinfo, + pGroupResInfo->currentGroup, endt - startt); + + _end: + tfree(pTableQueryInfoList); + tfree(posList); + tfree(pTree); + + return code; +} + +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo) { + int64_t st = taosGetTimestampUs(); + + while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { + SArray *group = GET_TABLEGROUP(pQInfo, pGroupResInfo->currentGroup); + + int32_t ret = mergeIntoGroupResultImpl(&pQInfo->runtimeEnv, pGroupResInfo, group, pQInfo); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + // this group generates at least one result, return results + if (taosArrayGetSize(pGroupResInfo->pRows) > 0) { + break; + } + + qDebug("QInfo:%p no result in group %d, continue", pQInfo, pGroupResInfo->currentGroup); + cleanupGroupResInfo(pGroupResInfo); + incNextGroup(pGroupResInfo); + } + + if (pGroupResInfo->currentGroup >= pGroupResInfo->totalGroup && !hasRemainData(pGroupResInfo)) { + SET_STABLE_QUERY_OVER(pQInfo); + } + + int64_t elapsedTime = taosGetTimestampUs() - st; + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, + pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); + + pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; + return TSDB_CODE_SUCCESS; +} diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c new file mode 100644 index 0000000000000000000000000000000000000000..e262a3ad385273f6e7f4b3c2cbe9c075e3d4e86d --- /dev/null +++ b/src/query/src/queryMain.c @@ -0,0 +1,542 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "qFill.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tglobal.h" + +#include "exception.h" +#include "hash.h" +#include "texpr.h" +#include "qExecutor.h" +#include "qResultbuf.h" +#include "qUtil.h" +#include "query.h" +#include "queryLog.h" +#include "tlosertree.h" +#include "ttype.h" +#include "tcompare.h" + +typedef struct SQueryMgmt { + pthread_mutex_t lock; + SCacheObj *qinfoPool; // query handle pool + int32_t vgId; + bool closed; +} SQueryMgmt; + +static void queryMgmtKillQueryFn(void* handle) { + void** fp = (void**)handle; + qKillQuery(*fp); +} + +static void freeqinfoFn(void *qhandle) { + void** handle = qhandle; + if (handle == NULL || *handle == NULL) { + return; + } + + qKillQuery(*handle); + qDestroyQueryInfo(*handle); +} + +void freeParam(SQueryParam *param) { + tfree(param->sql); + tfree(param->tagCond); + tfree(param->tbnameCond); + tfree(param->pTableIdList); + tfree(param->pExprMsg); + tfree(param->pSecExprMsg); + tfree(param->pExprs); + tfree(param->pSecExprs); + tfree(param->pGroupColIndex); + tfree(param->pTagColumnInfo); + tfree(param->pGroupbyExpr); + tfree(param->prevResult); +} + +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) { + assert(pQueryMsg != NULL && tsdb != NULL); + + int32_t code = TSDB_CODE_SUCCESS; + + SQueryParam param = {0}; + code = convertQueryMsg(pQueryMsg, ¶m); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (pQueryMsg->numOfTables <= 0) { + qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { + qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExprMsg, param.pTagColumnInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (param.pSecExprMsg != NULL) { + if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExprMsg, param.pTagColumnInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } + + param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); + if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { + goto _over; + } + + bool isSTableQuery = false; + STableGroupInfo tableGroupInfo = {0}; + int64_t st = taosGetTimestampUs(); + + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { + isSTableQuery = true; + + // also note there's possibility that only one table in the super table + if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } + + qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); + code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, + pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); + + if (code != TSDB_CODE_SUCCESS) { + qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); + goto _over; + } + } else { + code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + qDebug("qmsg:%p query on %" PRIzu " tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); + } + + int64_t el = taosGetTimestampUs() - st; + qDebug("qmsg:%p tag filter completed, numOfTables:%" PRIzu ", elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); + } else { + assert(0); + } + + code = checkForQueryBuf(tableGroupInfo.numOfTables); + if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort + goto _over; + } + + (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, param.pTagColumnInfo, isSTableQuery, param.sql); + + param.sql = NULL; + param.pExprs = NULL; + param.pSecExprs = NULL; + param.pGroupbyExpr = NULL; + param.pTagColumnInfo = NULL; + + if ((*pQInfo) == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _over; + } + + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, ¶m, isSTableQuery); + + _over: + if (param.pGroupbyExpr != NULL) { + taosArrayDestroy(param.pGroupbyExpr->columnInfo); + } + + taosArrayDestroy(param.pTableIdList); + param.pTableIdList = NULL; + + freeParam(¶m); + + for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { + SColumnInfo* column = pQueryMsg->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + + //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; + if (code != TSDB_CODE_SUCCESS) { + *pQInfo = NULL; + } + + // if failed to add ref for all tables in this query, abort current query + return code; +} + +bool qTableQuery(qinfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + assert(pQInfo && pQInfo->signature == pQInfo); + int64_t threadId = taosGetSelfPthreadId(); + + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { + qError("QInfo:%p qhandle is now executed by thread:%p", pQInfo, (void*) curOwner); + pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + return false; + } + + pQInfo->startExecTs = taosGetTimestampSec(); + + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:%p it is already killed, abort", pQInfo); + return doBuildResCheck(pQInfo); + } + + if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { + qDebug("QInfo:%p no table exists for query, abort", pQInfo); + setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); + return doBuildResCheck(pQInfo); + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pQInfo->runtimeEnv.env); + if (ret != TSDB_CODE_SUCCESS) { + pQInfo->code = ret; + qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); + return doBuildResCheck(pQInfo); + } + + qDebug("QInfo:%p query task is launched", pQInfo); + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { + assert(pQInfo->runtimeEnv.pQueryHandle == NULL); + buildTagQueryResult(pQInfo); + } else if (pQInfo->runtimeEnv.stableQuery) { + stableQueryImpl(pQInfo); + } else if (pQInfo->runtimeEnv.queryBlockDist){ + buildTableBlockDistResult(pQInfo); + } else { + tableQueryImpl(pQInfo); + } + + SQuery* pQuery = pRuntimeEnv->pQuery; + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:%p query is killed", pQInfo); + } else if (pQuery->rec.rows == 0) { + qDebug("QInfo:%p over, %" PRIzu " tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total); + } else { + qDebug("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", + pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + } + + return doBuildResCheck(pQInfo); +} + +int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + qError("QInfo:%p invalid qhandle", pQInfo); + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + *buildRes = false; + if (IS_QUERY_KILLED(pQInfo)) { + qDebug("QInfo:%p query is killed, code:0x%08x", pQInfo, pQInfo->code); + return pQInfo->code; + } + + int32_t code = TSDB_CODE_SUCCESS; + + if (tsRetrieveBlockingModel) { + pQInfo->rspContext = pRspContext; + tsem_wait(&pQInfo->ready); + *buildRes = true; + code = pQInfo->code; + } else { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + pthread_mutex_lock(&pQInfo->lock); + + assert(pQInfo->rspContext == NULL); + if (pQInfo->dataReady == QUERY_RESULT_READY) { + *buildRes = true; + qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%" PRId64 ", code:%s", pQInfo, pQuery->resultRowSize, + pQuery->rec.rows, tstrerror(pQInfo->code)); + } else { + *buildRes = false; + qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo); + pQInfo->rspContext = pRspContext; + assert(pQInfo->rspContext != NULL); + } + + code = pQInfo->code; + pthread_mutex_unlock(&pQInfo->lock); + } + + return code; +} + +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + + size += sizeof(int32_t); + size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo); + + *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); + + // current solution only avoid crash, but cannot return error code to client + *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); + if (*pRsp == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + (*pRsp)->numOfRows = htonl((int32_t)pQuery->rec.rows); + + if (pQInfo->code == TSDB_CODE_SUCCESS) { + (*pRsp)->offset = htobe64(pQuery->limit.offset); + (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); + } else { + (*pRsp)->offset = 0; + (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); + } + + (*pRsp)->precision = htons(pQuery->precision); + if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { + doDumpQueryResult(pQInfo, (*pRsp)->data); + } else { + setQueryStatus(pQuery, QUERY_OVER); + } + + pQInfo->rspContext = NULL; + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + + if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { + // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. + *continueExec = false; + (*pRsp)->completed = 1; // notify no more result to client + } else { + *continueExec = true; + qDebug("QInfo:%p has more results to retrieve", pQInfo); + } + + // the memory should be freed if the code of pQInfo is not TSDB_CODE_SUCCESS + if (pQInfo->code != TSDB_CODE_SUCCESS) { + rpcFreeCont(*pRsp); + *pRsp = NULL; + } + + return pQInfo->code; +} + +void* qGetResultRetrieveMsg(qinfo_t qinfo) { + SQInfo* pQInfo = (SQInfo*) qinfo; + assert(pQInfo != NULL); + + return pQInfo->rspContext; +} + +int32_t qKillQuery(qinfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + setQueryKilled(pQInfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while (pQInfo->owner != 0) { + taosMsleep(100); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qQueryCompleted(qinfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER); +} + +void qDestroyQueryInfo(qinfo_t qHandle) { + SQInfo* pQInfo = (SQInfo*) qHandle; + if (!isValidQInfo(pQInfo)) { + return; + } + + qDebug("QInfo:%p query completed", pQInfo); + queryCostStatis(pQInfo); // print the query cost summary + freeQInfo(pQInfo); +} + +void* qOpenQueryMgmt(int32_t vgId) { + const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool + + char cacheName[128] = {0}; + sprintf(cacheName, "qhandle_%d", vgId); + + SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + if (pQueryMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pQueryMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName); + pQueryMgmt->closed = false; + pQueryMgmt->vgId = vgId; + + pthread_mutex_init(&pQueryMgmt->lock, NULL); + + qDebug("vgId:%d, open querymgmt success", vgId); + return pQueryMgmt; +} + +void qQueryMgmtNotifyClosed(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + SQueryMgmt* pQueryMgmt = pQMgmt; + qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = true; + pthread_mutex_unlock(&pQueryMgmt->lock); + + taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); +} + +void qQueryMgmtReOpen(void *pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + SQueryMgmt *pQueryMgmt = pQMgmt; + qDebug("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = false; + pthread_mutex_unlock(&pQueryMgmt->lock); +} + +void qCleanupQueryMgmt(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + SQueryMgmt* pQueryMgmt = pQMgmt; + int32_t vgId = pQueryMgmt->vgId; + + assert(pQueryMgmt->closed); + + SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; + pQueryMgmt->qinfoPool = NULL; + + taosCacheCleanup(pqinfoPool); + pthread_mutex_destroy(&pQueryMgmt->lock); + tfree(pQueryMgmt); + + qDebug("vgId:%d, queryMgmt cleanup completed", vgId); +} + +void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { + if (pMgmt == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + SQueryMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + qError("QInfo:%p failed to add qhandle into qMgmt, since qMgmt is closed", (void *)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + pthread_mutex_lock(&pQueryMgmt->lock); + if (pQueryMgmt->closed) { + pthread_mutex_unlock(&pQueryMgmt->lock); + qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } else { + TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo; + void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), + (getMaximumIdleDurationSec()*1000)); + pthread_mutex_unlock(&pQueryMgmt->lock); + + return handle; + } +} + +void** qAcquireQInfo(void* pMgmt, uint64_t _key) { + SQueryMgmt *pQueryMgmt = pMgmt; + + if (pQueryMgmt->closed) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + if (pQueryMgmt->qinfoPool == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } + + TSDB_CACHE_PTR_TYPE key = (TSDB_CACHE_PTR_TYPE)_key; + void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &key, sizeof(TSDB_CACHE_PTR_TYPE)); + if (handle == NULL || *handle == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } else { + return handle; + } +} + +void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { + SQueryMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + return NULL; + } + + taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); + return 0; +} diff --git a/src/query/src/sql.c b/src/query/src/sql.c index d4e2b7f5f44677665d26f824506312b943060c40..2b1109688da3f7d814adea97665b718a30097a71 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -23,6 +23,7 @@ ** input grammar file: */ #include +#include /************ Begin %include sections from the grammar ************************/ #include @@ -76,8 +77,10 @@ ** zero the stack is dynamically sized using realloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser +** ParseCTX_* As ParseARG_ except for %extra_context ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -97,47 +100,56 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 274 +#define YYNOCODE 281 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - int yy46; - tSQLExpr* yy64; - tVariant yy134; - SCreateAcctSQL yy149; - SArray* yy165; - int64_t yy207; - SLimitVal yy216; - TAOS_FIELD yy223; - SSubclauseInfo* yy231; - SCreateDBInfo yy268; - tSQLExprList* yy290; - SQuerySQL* yy414; - SCreateTableSQL* yy470; - SIntervalVal yy532; + SCreatedTableInfo yy42; + SCreateAcctInfo yy47; + SQuerySQL* yy114; + TAOS_FIELD yy179; + SLimitVal yy204; + SSubclauseInfo* yy219; + int yy222; + SArray* yy247; + SCreateDbInfo yy262; + tSQLExpr* yy326; + SCreateTableSQL* yy358; + tVariant yy378; + int64_t yy403; + SIntervalVal yy430; + tSQLExprList* yy522; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif #define ParseARG_SDECL SSqlInfo* pInfo; #define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo -#define ParseARG_STORE yypParser->pInfo = pInfo +#define ParseARG_PARAM ,pInfo +#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; +#define ParseARG_STORE yypParser->pInfo=pInfo; +#define ParseCTX_SDECL +#define ParseCTX_PDECL +#define ParseCTX_PARAM +#define ParseCTX_FETCH +#define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 253 -#define YYNRULE 233 -#define YYNTOKEN 207 -#define YY_MAX_SHIFT 252 -#define YY_MIN_SHIFTREDUCE 420 -#define YY_MAX_SHIFTREDUCE 652 -#define YY_ERROR_ACTION 653 -#define YY_ACCEPT_ACTION 654 -#define YY_NO_ACTION 655 -#define YY_MIN_REDUCE 656 -#define YY_MAX_REDUCE 888 +#define YYNSTATE 294 +#define YYNRULE 254 +#define YYNRULE_WITH_ACTION 254 +#define YYNTOKEN 210 +#define YY_MAX_SHIFT 293 +#define YY_MIN_SHIFTREDUCE 477 +#define YY_MAX_SHIFTREDUCE 730 +#define YY_ERROR_ACTION 731 +#define YY_ACCEPT_ACTION 732 +#define YY_NO_ACTION 733 +#define YY_MIN_REDUCE 734 +#define YY_MAX_REDUCE 987 /************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -202,224 +214,252 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (571) +#define YY_ACTTAB_COUNT (651) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 108, 463, 141, 11, 654, 252, 802, 463, 140, 464, - /* 10 */ 162, 165, 876, 35, 36, 464, 37, 38, 159, 250, - /* 20 */ 170, 29, 141, 463, 206, 41, 39, 43, 40, 173, - /* 30 */ 780, 464, 875, 34, 33, 145, 141, 32, 31, 30, - /* 40 */ 35, 36, 791, 37, 38, 164, 876, 170, 29, 780, - /* 50 */ 21, 206, 41, 39, 43, 40, 191, 829, 799, 201, - /* 60 */ 34, 33, 21, 21, 32, 31, 30, 421, 422, 423, - /* 70 */ 424, 425, 426, 427, 428, 429, 430, 431, 432, 251, - /* 80 */ 35, 36, 181, 37, 38, 532, 776, 170, 29, 238, - /* 90 */ 246, 206, 41, 39, 43, 40, 174, 175, 777, 777, - /* 100 */ 34, 33, 872, 56, 32, 31, 30, 176, 871, 36, - /* 110 */ 780, 37, 38, 227, 226, 170, 29, 791, 17, 206, - /* 120 */ 41, 39, 43, 40, 108, 26, 870, 606, 34, 33, - /* 130 */ 78, 160, 32, 31, 30, 238, 157, 16, 218, 245, - /* 140 */ 244, 217, 216, 215, 243, 214, 242, 241, 240, 213, - /* 150 */ 239, 755, 103, 743, 744, 745, 746, 747, 748, 749, - /* 160 */ 750, 751, 752, 753, 754, 756, 37, 38, 229, 177, - /* 170 */ 170, 29, 224, 223, 206, 41, 39, 43, 40, 203, - /* 180 */ 62, 60, 8, 34, 33, 63, 118, 32, 31, 30, - /* 190 */ 169, 619, 27, 12, 610, 184, 613, 158, 616, 778, - /* 200 */ 169, 619, 188, 187, 610, 194, 613, 108, 616, 153, - /* 210 */ 169, 619, 561, 108, 610, 154, 613, 18, 616, 90, - /* 220 */ 89, 148, 166, 167, 34, 33, 205, 143, 32, 31, - /* 230 */ 30, 697, 166, 167, 131, 144, 564, 41, 39, 43, - /* 240 */ 40, 706, 166, 167, 131, 34, 33, 146, 17, 32, - /* 250 */ 31, 30, 32, 31, 30, 26, 16, 207, 245, 244, - /* 260 */ 21, 587, 588, 243, 828, 242, 241, 240, 698, 239, - /* 270 */ 61, 131, 76, 80, 147, 190, 102, 151, 85, 88, - /* 280 */ 79, 760, 156, 26, 758, 759, 82, 21, 42, 761, - /* 290 */ 556, 763, 764, 762, 225, 765, 777, 193, 42, 618, - /* 300 */ 249, 248, 96, 574, 121, 122, 608, 105, 42, 618, - /* 310 */ 70, 66, 69, 578, 617, 168, 579, 46, 152, 618, - /* 320 */ 14, 230, 548, 777, 617, 545, 638, 546, 150, 547, - /* 330 */ 13, 135, 133, 612, 617, 615, 139, 93, 92, 91, - /* 340 */ 620, 611, 609, 614, 13, 47, 538, 622, 50, 552, - /* 350 */ 46, 553, 537, 178, 179, 3, 22, 211, 75, 74, - /* 360 */ 149, 22, 10, 9, 48, 51, 142, 550, 885, 551, - /* 370 */ 87, 86, 101, 99, 779, 839, 838, 171, 835, 834, - /* 380 */ 172, 801, 771, 228, 806, 793, 808, 104, 821, 119, - /* 390 */ 820, 117, 120, 708, 212, 137, 24, 221, 705, 222, - /* 400 */ 26, 192, 100, 884, 72, 883, 881, 123, 726, 25, - /* 410 */ 573, 23, 138, 695, 49, 81, 693, 83, 84, 691, - /* 420 */ 790, 690, 195, 161, 199, 549, 57, 52, 180, 132, - /* 430 */ 688, 687, 686, 685, 684, 134, 682, 109, 680, 678, - /* 440 */ 44, 676, 674, 136, 204, 202, 58, 822, 200, 198, - /* 450 */ 196, 220, 77, 28, 231, 232, 233, 235, 652, 234, - /* 460 */ 236, 237, 247, 209, 183, 53, 651, 182, 185, 186, - /* 470 */ 64, 67, 155, 650, 643, 189, 193, 689, 558, 94, - /* 480 */ 683, 675, 126, 125, 727, 129, 124, 127, 128, 95, - /* 490 */ 130, 1, 114, 110, 111, 775, 2, 55, 59, 116, - /* 500 */ 112, 113, 115, 575, 106, 163, 197, 5, 580, 107, - /* 510 */ 6, 65, 621, 19, 4, 20, 15, 208, 623, 7, - /* 520 */ 210, 504, 500, 498, 497, 496, 493, 467, 219, 68, - /* 530 */ 45, 71, 73, 22, 534, 533, 531, 488, 54, 486, - /* 540 */ 478, 484, 480, 482, 476, 474, 505, 503, 502, 501, - /* 550 */ 499, 495, 494, 46, 465, 436, 434, 656, 655, 655, - /* 560 */ 655, 655, 655, 655, 655, 655, 655, 655, 655, 97, - /* 570 */ 98, + /* 0 */ 74, 521, 732, 293, 521, 165, 186, 291, 28, 522, + /* 10 */ 190, 893, 522, 43, 44, 969, 47, 48, 15, 776, + /* 20 */ 198, 37, 152, 46, 242, 51, 49, 53, 50, 854, + /* 30 */ 855, 27, 858, 42, 41, 871, 128, 40, 39, 38, + /* 40 */ 43, 44, 882, 47, 48, 882, 188, 198, 37, 868, + /* 50 */ 46, 242, 51, 49, 53, 50, 187, 128, 203, 225, + /* 60 */ 42, 41, 979, 165, 40, 39, 38, 43, 44, 890, + /* 70 */ 47, 48, 193, 970, 198, 37, 165, 46, 242, 51, + /* 80 */ 49, 53, 50, 871, 128, 192, 970, 42, 41, 258, + /* 90 */ 521, 40, 39, 38, 290, 289, 115, 239, 522, 71, + /* 100 */ 77, 43, 45, 128, 47, 48, 205, 66, 198, 37, + /* 110 */ 28, 46, 242, 51, 49, 53, 50, 40, 39, 38, + /* 120 */ 921, 42, 41, 278, 65, 40, 39, 38, 865, 678, + /* 130 */ 287, 871, 859, 210, 478, 479, 480, 481, 482, 483, + /* 140 */ 484, 485, 486, 487, 488, 489, 292, 72, 201, 215, + /* 150 */ 44, 868, 47, 48, 856, 871, 198, 37, 209, 46, + /* 160 */ 242, 51, 49, 53, 50, 869, 922, 204, 237, 42, + /* 170 */ 41, 96, 163, 40, 39, 38, 278, 21, 256, 286, + /* 180 */ 285, 255, 254, 253, 284, 252, 283, 282, 281, 251, + /* 190 */ 280, 279, 835, 594, 823, 824, 825, 826, 827, 828, + /* 200 */ 829, 830, 831, 832, 833, 834, 836, 837, 47, 48, + /* 210 */ 87, 86, 198, 37, 28, 46, 242, 51, 49, 53, + /* 220 */ 50, 268, 267, 16, 211, 42, 41, 265, 264, 40, + /* 230 */ 39, 38, 197, 691, 28, 634, 682, 207, 685, 174, + /* 240 */ 688, 22, 42, 41, 73, 175, 40, 39, 38, 34, + /* 250 */ 108, 107, 173, 197, 691, 867, 67, 682, 28, 685, + /* 260 */ 21, 688, 286, 285, 194, 195, 169, 284, 241, 283, + /* 270 */ 282, 281, 202, 280, 279, 868, 618, 28, 60, 615, + /* 280 */ 22, 616, 631, 617, 218, 194, 195, 123, 34, 23, + /* 290 */ 841, 222, 221, 839, 840, 857, 261, 61, 842, 868, + /* 300 */ 844, 845, 843, 208, 846, 847, 260, 212, 213, 224, + /* 310 */ 638, 51, 49, 53, 50, 262, 181, 28, 868, 42, + /* 320 */ 41, 94, 98, 40, 39, 38, 28, 88, 103, 106, + /* 330 */ 97, 10, 52, 3, 142, 76, 100, 138, 680, 31, + /* 340 */ 83, 79, 82, 158, 154, 690, 230, 659, 660, 156, + /* 350 */ 111, 110, 109, 52, 785, 266, 777, 152, 868, 152, + /* 360 */ 689, 626, 121, 684, 270, 687, 690, 868, 196, 227, + /* 370 */ 34, 228, 258, 646, 681, 29, 683, 125, 686, 650, + /* 380 */ 651, 689, 619, 56, 18, 711, 692, 243, 966, 17, + /* 390 */ 17, 57, 604, 245, 606, 247, 29, 29, 56, 75, + /* 400 */ 605, 63, 26, 593, 56, 248, 12, 11, 93, 92, + /* 410 */ 4, 965, 58, 14, 13, 622, 620, 623, 621, 105, + /* 420 */ 104, 120, 118, 932, 964, 182, 183, 167, 168, 170, + /* 430 */ 164, 171, 172, 178, 179, 177, 162, 176, 166, 870, + /* 440 */ 931, 199, 928, 927, 884, 200, 269, 122, 892, 35, + /* 450 */ 899, 901, 124, 139, 864, 914, 140, 141, 913, 137, + /* 460 */ 787, 250, 160, 32, 259, 34, 784, 984, 84, 983, + /* 470 */ 981, 143, 226, 119, 231, 263, 978, 90, 977, 975, + /* 480 */ 694, 144, 645, 805, 189, 235, 62, 881, 129, 33, + /* 490 */ 59, 240, 30, 54, 161, 132, 130, 238, 236, 131, + /* 500 */ 774, 99, 772, 133, 234, 134, 232, 101, 36, 102, + /* 510 */ 770, 95, 769, 271, 272, 214, 153, 767, 766, 765, + /* 520 */ 764, 763, 273, 155, 157, 760, 758, 756, 754, 752, + /* 530 */ 159, 274, 229, 68, 69, 915, 275, 276, 277, 184, + /* 540 */ 206, 249, 730, 185, 180, 288, 80, 216, 217, 768, + /* 550 */ 729, 220, 219, 112, 728, 762, 147, 761, 146, 806, + /* 560 */ 145, 148, 149, 151, 150, 113, 114, 753, 1, 716, + /* 570 */ 2, 223, 227, 628, 64, 6, 866, 244, 70, 647, + /* 580 */ 126, 135, 136, 191, 24, 233, 7, 652, 127, 8, + /* 590 */ 693, 5, 25, 9, 19, 246, 20, 695, 78, 76, + /* 600 */ 562, 558, 556, 555, 554, 551, 525, 257, 81, 85, + /* 610 */ 29, 55, 596, 595, 89, 91, 592, 546, 544, 536, + /* 620 */ 542, 538, 540, 534, 532, 564, 563, 561, 560, 559, + /* 630 */ 557, 553, 552, 56, 523, 493, 491, 734, 733, 733, + /* 640 */ 733, 733, 733, 733, 733, 733, 733, 733, 733, 116, + /* 650 */ 117, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 211, 1, 262, 262, 208, 209, 211, 1, 262, 9, - /* 10 */ 228, 271, 272, 13, 14, 9, 16, 17, 210, 211, - /* 20 */ 20, 21, 262, 1, 24, 25, 26, 27, 28, 228, - /* 30 */ 248, 9, 272, 33, 34, 262, 262, 37, 38, 39, - /* 40 */ 13, 14, 246, 16, 17, 271, 272, 20, 21, 248, - /* 50 */ 211, 24, 25, 26, 27, 28, 260, 268, 263, 270, - /* 60 */ 33, 34, 211, 211, 37, 38, 39, 45, 46, 47, - /* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 80 */ 13, 14, 60, 16, 17, 5, 247, 20, 21, 78, - /* 90 */ 228, 24, 25, 26, 27, 28, 245, 245, 247, 247, - /* 100 */ 33, 34, 262, 103, 37, 38, 39, 66, 262, 14, - /* 110 */ 248, 16, 17, 33, 34, 20, 21, 246, 99, 24, - /* 120 */ 25, 26, 27, 28, 211, 106, 262, 100, 33, 34, - /* 130 */ 73, 260, 37, 38, 39, 78, 262, 85, 86, 87, - /* 140 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - /* 150 */ 98, 227, 211, 229, 230, 231, 232, 233, 234, 235, - /* 160 */ 236, 237, 238, 239, 240, 241, 16, 17, 211, 128, - /* 170 */ 20, 21, 131, 132, 24, 25, 26, 27, 28, 266, - /* 180 */ 249, 268, 99, 33, 34, 102, 103, 37, 38, 39, - /* 190 */ 1, 2, 261, 44, 5, 127, 7, 262, 9, 242, - /* 200 */ 1, 2, 134, 135, 5, 264, 7, 211, 9, 60, - /* 210 */ 1, 2, 104, 211, 5, 66, 7, 109, 9, 70, - /* 220 */ 71, 72, 33, 34, 33, 34, 37, 262, 37, 38, - /* 230 */ 39, 215, 33, 34, 218, 262, 37, 25, 26, 27, - /* 240 */ 28, 215, 33, 34, 218, 33, 34, 262, 99, 37, - /* 250 */ 38, 39, 37, 38, 39, 106, 85, 15, 87, 88, - /* 260 */ 211, 116, 117, 92, 268, 94, 95, 96, 215, 98, - /* 270 */ 268, 218, 61, 62, 262, 126, 99, 262, 67, 68, - /* 280 */ 69, 227, 133, 106, 230, 231, 75, 211, 99, 235, - /* 290 */ 100, 237, 238, 239, 245, 241, 247, 107, 99, 110, - /* 300 */ 63, 64, 65, 100, 61, 62, 1, 104, 99, 110, - /* 310 */ 67, 68, 69, 100, 125, 59, 100, 104, 262, 110, - /* 320 */ 104, 245, 2, 247, 125, 5, 100, 7, 262, 9, - /* 330 */ 104, 61, 62, 5, 125, 7, 262, 67, 68, 69, - /* 340 */ 100, 5, 37, 7, 104, 104, 100, 105, 104, 5, - /* 350 */ 104, 7, 100, 33, 34, 99, 104, 100, 129, 130, - /* 360 */ 262, 104, 129, 130, 123, 121, 262, 5, 248, 7, - /* 370 */ 73, 74, 61, 62, 248, 243, 243, 243, 243, 243, - /* 380 */ 243, 211, 244, 243, 211, 246, 211, 211, 269, 211, - /* 390 */ 269, 250, 211, 211, 211, 211, 211, 211, 211, 211, - /* 400 */ 106, 246, 59, 211, 211, 211, 211, 211, 211, 211, - /* 410 */ 110, 211, 211, 211, 122, 211, 211, 211, 211, 211, - /* 420 */ 259, 211, 265, 265, 265, 105, 212, 120, 211, 211, - /* 430 */ 211, 211, 211, 211, 211, 211, 211, 258, 211, 211, - /* 440 */ 119, 211, 211, 211, 114, 118, 212, 212, 113, 112, - /* 450 */ 111, 76, 84, 124, 83, 49, 80, 53, 5, 82, - /* 460 */ 81, 79, 76, 212, 5, 212, 5, 136, 136, 5, - /* 470 */ 216, 216, 212, 5, 86, 127, 107, 212, 100, 213, - /* 480 */ 212, 212, 220, 224, 226, 222, 225, 223, 221, 213, - /* 490 */ 219, 217, 253, 257, 256, 246, 214, 108, 104, 251, - /* 500 */ 255, 254, 252, 100, 99, 1, 99, 115, 100, 99, - /* 510 */ 115, 73, 100, 104, 99, 104, 99, 101, 105, 99, - /* 520 */ 101, 9, 5, 5, 5, 5, 5, 77, 15, 73, - /* 530 */ 16, 130, 130, 104, 5, 5, 100, 5, 99, 5, - /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 550 */ 5, 5, 5, 104, 77, 59, 58, 0, 273, 273, - /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 21, - /* 570 */ 21, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 580 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 590 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 600 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 610 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 620 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 630 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 640 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 650 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 660 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 670 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 680 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 690 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 700 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 710 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 720 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 730 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 740 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 760 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 770 */ 273, 273, 273, 273, 273, 273, 273, 273, + /* 0 */ 218, 1, 210, 211, 1, 270, 212, 213, 213, 9, + /* 10 */ 230, 213, 9, 13, 14, 280, 16, 17, 270, 217, + /* 20 */ 20, 21, 220, 23, 24, 25, 26, 27, 28, 247, + /* 30 */ 248, 249, 250, 33, 34, 255, 213, 37, 38, 39, + /* 40 */ 13, 14, 253, 16, 17, 253, 251, 20, 21, 254, + /* 50 */ 23, 24, 25, 26, 27, 28, 267, 213, 230, 267, + /* 60 */ 33, 34, 255, 270, 37, 38, 39, 13, 14, 271, + /* 70 */ 16, 17, 279, 280, 20, 21, 270, 23, 24, 25, + /* 80 */ 26, 27, 28, 255, 213, 279, 280, 33, 34, 77, + /* 90 */ 1, 37, 38, 39, 63, 64, 65, 274, 9, 276, + /* 100 */ 218, 13, 14, 213, 16, 17, 230, 107, 20, 21, + /* 110 */ 213, 23, 24, 25, 26, 27, 28, 37, 38, 39, + /* 120 */ 276, 33, 34, 79, 218, 37, 38, 39, 213, 102, + /* 130 */ 230, 255, 250, 213, 45, 46, 47, 48, 49, 50, + /* 140 */ 51, 52, 53, 54, 55, 56, 57, 276, 251, 60, + /* 150 */ 14, 254, 16, 17, 248, 255, 20, 21, 66, 23, + /* 160 */ 24, 25, 26, 27, 28, 245, 276, 252, 278, 33, + /* 170 */ 34, 74, 270, 37, 38, 39, 79, 86, 87, 88, + /* 180 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, + /* 190 */ 99, 100, 229, 5, 231, 232, 233, 234, 235, 236, + /* 200 */ 237, 238, 239, 240, 241, 242, 243, 244, 16, 17, + /* 210 */ 133, 134, 20, 21, 213, 23, 24, 25, 26, 27, + /* 220 */ 28, 33, 34, 44, 132, 33, 34, 135, 136, 37, + /* 230 */ 38, 39, 1, 2, 213, 37, 5, 66, 7, 60, + /* 240 */ 9, 101, 33, 34, 256, 66, 37, 38, 39, 109, + /* 250 */ 71, 72, 73, 1, 2, 254, 268, 5, 213, 7, + /* 260 */ 86, 9, 88, 89, 33, 34, 270, 93, 37, 95, + /* 270 */ 96, 97, 251, 99, 100, 254, 2, 213, 106, 5, + /* 280 */ 101, 7, 106, 9, 131, 33, 34, 213, 109, 113, + /* 290 */ 229, 138, 139, 232, 233, 0, 251, 125, 237, 254, + /* 300 */ 239, 240, 241, 132, 243, 244, 135, 33, 34, 130, + /* 310 */ 112, 25, 26, 27, 28, 251, 137, 213, 254, 33, + /* 320 */ 34, 61, 62, 37, 38, 39, 213, 67, 68, 69, + /* 330 */ 70, 101, 101, 61, 62, 105, 76, 107, 1, 67, + /* 340 */ 68, 69, 70, 61, 62, 114, 272, 120, 121, 67, + /* 350 */ 68, 69, 70, 101, 217, 251, 217, 220, 254, 220, + /* 360 */ 129, 102, 101, 5, 251, 7, 114, 254, 59, 110, + /* 370 */ 109, 102, 77, 102, 37, 106, 5, 106, 7, 102, + /* 380 */ 102, 129, 108, 106, 106, 102, 102, 15, 270, 106, + /* 390 */ 106, 106, 102, 102, 102, 102, 106, 106, 106, 106, + /* 400 */ 102, 101, 101, 103, 106, 104, 133, 134, 133, 134, + /* 410 */ 101, 270, 127, 133, 134, 5, 5, 7, 7, 74, + /* 420 */ 75, 61, 62, 246, 270, 270, 270, 270, 270, 270, + /* 430 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 255, + /* 440 */ 246, 246, 246, 246, 253, 246, 246, 213, 213, 269, + /* 450 */ 213, 213, 213, 213, 213, 277, 213, 213, 277, 257, + /* 460 */ 213, 213, 213, 213, 213, 109, 213, 213, 213, 213, + /* 470 */ 213, 213, 253, 59, 273, 213, 213, 213, 213, 213, + /* 480 */ 108, 213, 114, 213, 273, 273, 124, 266, 265, 213, + /* 490 */ 126, 118, 213, 123, 213, 262, 264, 122, 117, 263, + /* 500 */ 213, 213, 213, 261, 116, 260, 115, 213, 128, 213, + /* 510 */ 213, 85, 213, 84, 49, 213, 213, 213, 213, 213, + /* 520 */ 213, 213, 81, 213, 213, 213, 213, 213, 213, 213, + /* 530 */ 213, 83, 214, 214, 214, 214, 53, 82, 80, 214, + /* 540 */ 214, 214, 5, 214, 214, 77, 218, 140, 5, 214, + /* 550 */ 5, 5, 140, 215, 5, 214, 222, 214, 226, 228, + /* 560 */ 227, 225, 223, 221, 224, 215, 215, 214, 219, 87, + /* 570 */ 216, 131, 110, 102, 111, 101, 253, 104, 106, 102, + /* 580 */ 101, 259, 258, 1, 106, 101, 119, 102, 101, 119, + /* 590 */ 102, 101, 106, 101, 101, 104, 101, 108, 74, 105, + /* 600 */ 9, 5, 5, 5, 5, 5, 78, 15, 74, 134, + /* 610 */ 106, 16, 5, 5, 134, 134, 102, 5, 5, 5, + /* 620 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 630 */ 5, 5, 5, 106, 78, 59, 58, 0, 281, 281, + /* 640 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 21, + /* 650 */ 21, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 660 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 670 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 680 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 690 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 700 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 710 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 720 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 730 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 740 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 750 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 760 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 770 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 780 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 790 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 800 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 810 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 820 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 830 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 840 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 850 */ 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + /* 860 */ 281, }; -#define YY_SHIFT_COUNT (252) +#define YY_SHIFT_COUNT (293) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (557) +#define YY_SHIFT_MAX (637) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 149, 52, 171, 189, 209, 6, 6, 6, 6, 6, - /* 10 */ 6, 0, 22, 209, 320, 320, 320, 19, 6, 6, - /* 20 */ 6, 6, 6, 57, 11, 11, 571, 199, 209, 209, - /* 30 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, - /* 40 */ 209, 209, 209, 209, 209, 320, 320, 80, 80, 80, - /* 50 */ 80, 80, 80, 83, 80, 177, 6, 6, 6, 6, - /* 60 */ 145, 145, 108, 6, 6, 6, 6, 6, 6, 6, - /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 100 */ 6, 6, 294, 343, 343, 300, 300, 300, 343, 307, - /* 110 */ 292, 321, 330, 327, 335, 337, 339, 329, 294, 343, - /* 120 */ 343, 375, 375, 343, 368, 371, 406, 376, 377, 404, - /* 130 */ 379, 382, 343, 386, 343, 386, 343, 571, 571, 27, - /* 140 */ 67, 67, 67, 95, 150, 212, 212, 212, 211, 191, - /* 150 */ 191, 191, 191, 243, 270, 41, 68, 215, 215, 237, - /* 160 */ 190, 203, 213, 216, 226, 240, 328, 336, 305, 256, - /* 170 */ 242, 241, 244, 246, 252, 257, 229, 233, 344, 362, - /* 180 */ 297, 311, 453, 331, 459, 461, 332, 464, 468, 388, - /* 190 */ 348, 369, 378, 389, 394, 403, 405, 504, 407, 408, - /* 200 */ 410, 409, 392, 411, 395, 412, 415, 413, 417, 416, - /* 210 */ 420, 419, 438, 512, 517, 518, 519, 520, 521, 450, - /* 220 */ 513, 456, 514, 401, 402, 429, 529, 530, 436, 439, - /* 230 */ 429, 532, 534, 535, 536, 537, 538, 539, 540, 541, - /* 240 */ 542, 543, 544, 545, 546, 547, 449, 477, 548, 549, - /* 250 */ 496, 498, 557, + /* 0 */ 179, 91, 174, 12, 231, 252, 3, 3, 3, 3, + /* 10 */ 3, 3, 3, 3, 3, 0, 89, 252, 274, 274, + /* 20 */ 274, 274, 140, 3, 3, 3, 3, 295, 3, 3, + /* 30 */ 97, 12, 44, 44, 651, 252, 252, 252, 252, 252, + /* 40 */ 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, + /* 50 */ 252, 252, 252, 252, 252, 274, 274, 188, 188, 188, + /* 60 */ 188, 188, 188, 188, 261, 3, 3, 198, 3, 3, + /* 70 */ 3, 227, 227, 176, 3, 3, 3, 3, 3, 3, + /* 80 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + /* 90 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + /* 100 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + /* 110 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + /* 120 */ 3, 356, 414, 414, 414, 368, 368, 368, 414, 362, + /* 130 */ 364, 370, 373, 375, 381, 388, 391, 380, 356, 414, + /* 140 */ 414, 414, 12, 414, 414, 426, 429, 465, 441, 448, + /* 150 */ 483, 455, 458, 414, 468, 414, 468, 414, 468, 414, + /* 160 */ 651, 651, 27, 54, 88, 54, 54, 136, 192, 286, + /* 170 */ 286, 286, 286, 260, 272, 282, 209, 209, 209, 209, + /* 180 */ 92, 153, 80, 80, 230, 171, 31, 259, 269, 271, + /* 190 */ 277, 278, 283, 284, 358, 371, 337, 309, 372, 285, + /* 200 */ 172, 290, 291, 292, 293, 298, 301, 77, 273, 275, + /* 210 */ 300, 280, 410, 411, 345, 360, 537, 407, 543, 545, + /* 220 */ 412, 546, 549, 482, 440, 462, 471, 463, 473, 474, + /* 230 */ 472, 477, 479, 582, 484, 485, 487, 478, 467, 486, + /* 240 */ 470, 488, 490, 489, 492, 473, 493, 491, 495, 494, + /* 250 */ 524, 591, 596, 597, 598, 599, 600, 528, 592, 534, + /* 260 */ 475, 504, 504, 595, 480, 481, 504, 607, 608, 514, + /* 270 */ 504, 612, 613, 614, 615, 616, 617, 618, 619, 620, + /* 280 */ 621, 622, 623, 624, 625, 626, 627, 527, 556, 628, + /* 290 */ 629, 576, 578, 637, }; -#define YY_REDUCE_COUNT (138) -#define YY_REDUCE_MIN (-260) -#define YY_REDUCE_MAX (282) +#define YY_REDUCE_COUNT (161) +#define YY_REDUCE_MIN (-265) +#define YY_REDUCE_MAX (354) static const short yy_reduce_ofst[] = { - /* 0 */ -204, -76, 54, -260, -226, -211, -87, -149, -148, 49, - /* 10 */ 76, -205, -192, -240, -218, -199, -138, -129, -59, -4, - /* 20 */ 2, -43, -161, 16, 26, 53, -69, -259, -254, -227, - /* 30 */ -160, -154, -136, -126, -65, -35, -27, -15, 12, 15, - /* 40 */ 56, 66, 74, 98, 104, 120, 126, 132, 133, 134, - /* 50 */ 135, 136, 137, 138, 140, 139, 170, 173, 175, 176, - /* 60 */ 119, 121, 141, 178, 181, 182, 183, 184, 185, 186, - /* 70 */ 187, 188, 192, 193, 194, 195, 196, 197, 198, 200, - /* 80 */ 201, 202, 204, 205, 206, 207, 208, 210, 217, 218, - /* 90 */ 219, 220, 221, 222, 223, 224, 225, 227, 228, 230, - /* 100 */ 231, 232, 155, 214, 234, 157, 158, 159, 235, 161, - /* 110 */ 179, 236, 238, 245, 247, 239, 250, 248, 249, 251, - /* 120 */ 253, 254, 255, 260, 258, 261, 259, 262, 264, 267, - /* 130 */ 263, 271, 265, 266, 268, 276, 269, 274, 282, + /* 0 */ -208, -37, 61, -218, -207, -194, -205, -110, -177, -103, + /* 10 */ 21, 45, 64, 104, 113, -202, -206, -265, -220, -172, + /* 20 */ -124, -100, -211, 74, -156, -129, -85, -118, -80, 1, + /* 30 */ -198, -94, 137, 139, -12, -252, -98, -4, 118, 141, + /* 40 */ 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, + /* 50 */ 164, 165, 166, 167, 168, -193, 184, 177, 194, 195, + /* 60 */ 196, 197, 199, 200, 191, 234, 235, 180, 237, 238, + /* 70 */ 239, 178, 181, 202, 240, 241, 243, 244, 247, 248, + /* 80 */ 249, 250, 251, 253, 254, 255, 256, 257, 258, 262, + /* 90 */ 263, 264, 265, 266, 268, 270, 276, 279, 281, 287, + /* 100 */ 288, 289, 294, 296, 297, 299, 302, 303, 304, 305, + /* 110 */ 306, 307, 308, 310, 311, 312, 313, 314, 315, 316, + /* 120 */ 317, 219, 318, 319, 320, 201, 211, 212, 321, 221, + /* 130 */ 223, 232, 236, 233, 242, 245, 322, 324, 323, 325, + /* 140 */ 326, 327, 328, 329, 330, 331, 333, 332, 334, 336, + /* 150 */ 339, 340, 342, 335, 338, 341, 350, 343, 351, 353, + /* 160 */ 349, 354, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 653, 707, 696, 878, 878, 653, 653, 653, 653, 653, - /* 10 */ 653, 803, 671, 878, 653, 653, 653, 653, 653, 653, - /* 20 */ 653, 653, 653, 709, 709, 709, 798, 653, 653, 653, - /* 30 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 40 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 50 */ 653, 653, 653, 653, 653, 653, 653, 805, 807, 653, - /* 60 */ 825, 825, 796, 653, 653, 653, 653, 653, 653, 653, - /* 70 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 80 */ 653, 694, 653, 692, 653, 653, 653, 653, 653, 653, - /* 90 */ 653, 653, 653, 653, 653, 653, 681, 653, 653, 653, - /* 100 */ 653, 653, 653, 673, 673, 653, 653, 653, 673, 832, - /* 110 */ 836, 830, 818, 826, 817, 813, 812, 840, 653, 673, - /* 120 */ 673, 704, 704, 673, 725, 723, 721, 713, 719, 715, - /* 130 */ 717, 711, 673, 702, 673, 702, 673, 742, 757, 653, - /* 140 */ 841, 877, 831, 867, 866, 873, 865, 864, 653, 860, - /* 150 */ 861, 863, 862, 653, 653, 653, 653, 869, 868, 653, - /* 160 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 843, - /* 170 */ 653, 837, 833, 653, 653, 653, 653, 653, 653, 653, - /* 180 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 190 */ 653, 795, 653, 653, 804, 653, 653, 653, 653, 653, - /* 200 */ 653, 827, 653, 819, 653, 653, 653, 653, 653, 653, - /* 210 */ 653, 772, 653, 653, 653, 653, 653, 653, 653, 653, - /* 220 */ 653, 653, 653, 653, 653, 882, 653, 653, 653, 766, - /* 230 */ 880, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 240 */ 653, 653, 653, 653, 653, 653, 728, 653, 679, 677, - /* 250 */ 653, 669, 653, + /* 0 */ 731, 786, 775, 783, 972, 972, 731, 731, 731, 731, + /* 10 */ 731, 731, 731, 731, 731, 894, 749, 972, 731, 731, + /* 20 */ 731, 731, 731, 731, 731, 731, 731, 783, 731, 731, + /* 30 */ 788, 783, 788, 788, 889, 731, 731, 731, 731, 731, + /* 40 */ 731, 731, 731, 731, 731, 731, 731, 731, 731, 731, + /* 50 */ 731, 731, 731, 731, 731, 731, 731, 731, 731, 731, + /* 60 */ 731, 731, 731, 731, 731, 731, 731, 896, 898, 900, + /* 70 */ 731, 918, 918, 887, 731, 731, 731, 731, 731, 731, + /* 80 */ 731, 731, 731, 731, 731, 731, 731, 731, 731, 731, + /* 90 */ 731, 731, 731, 731, 731, 731, 731, 731, 731, 773, + /* 100 */ 731, 771, 731, 731, 731, 731, 731, 731, 731, 731, + /* 110 */ 731, 731, 731, 731, 731, 759, 731, 731, 731, 731, + /* 120 */ 731, 731, 751, 751, 751, 731, 731, 731, 751, 925, + /* 130 */ 929, 923, 911, 919, 910, 906, 905, 933, 731, 751, + /* 140 */ 751, 751, 783, 751, 751, 804, 802, 800, 792, 798, + /* 150 */ 794, 796, 790, 751, 781, 751, 781, 751, 781, 751, + /* 160 */ 822, 838, 731, 934, 731, 971, 924, 961, 960, 967, + /* 170 */ 959, 958, 957, 731, 731, 731, 953, 954, 956, 955, + /* 180 */ 731, 731, 963, 962, 731, 731, 731, 731, 731, 731, + /* 190 */ 731, 731, 731, 731, 731, 731, 731, 936, 731, 930, + /* 200 */ 926, 731, 731, 731, 731, 731, 731, 731, 731, 731, + /* 210 */ 848, 731, 731, 731, 731, 731, 731, 731, 731, 731, + /* 220 */ 731, 731, 731, 731, 731, 886, 731, 731, 731, 731, + /* 230 */ 897, 731, 731, 731, 731, 731, 731, 920, 731, 912, + /* 240 */ 731, 731, 731, 731, 731, 860, 731, 731, 731, 731, + /* 250 */ 731, 731, 731, 731, 731, 731, 731, 731, 731, 731, + /* 260 */ 731, 982, 980, 731, 731, 731, 976, 731, 731, 731, + /* 270 */ 974, 731, 731, 731, 731, 731, 731, 731, 731, 731, + /* 280 */ 731, 731, 731, 731, 731, 731, 731, 807, 731, 757, + /* 290 */ 755, 731, 747, 731, }; /********** End of lemon-generated parsing tables *****************************/ @@ -506,6 +546,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* STABLES => nothing */ 0, /* VGROUPS => nothing */ 0, /* DROP => nothing */ + 1, /* STABLE => ID */ 0, /* DNODE => nothing */ 0, /* USER => nothing */ 0, /* ACCOUNT => nothing */ @@ -538,16 +579,19 @@ static const YYCODETYPE yyFallback[] = { 0, /* COMP => nothing */ 0, /* PRECISION => nothing */ 0, /* UPDATE => nothing */ + 0, /* CACHELAST => nothing */ 0, /* LP => nothing */ 0, /* RP => nothing */ + 0, /* UNSIGNED => nothing */ 0, /* TAGS => nothing */ 0, /* USING => nothing */ - 0, /* AS => nothing */ 0, /* COMMA => nothing */ + 0, /* AS => nothing */ 1, /* NULL => ID */ 0, /* SELECT => nothing */ 0, /* UNION => nothing */ 1, /* ALL => ID */ + 0, /* DISTINCT => nothing */ 0, /* FROM => nothing */ 0, /* VARIABLE => nothing */ 0, /* INTERVAL => nothing */ @@ -642,7 +686,6 @@ static const YYCODETYPE yyFallback[] = { 1, /* TBNAME => ID */ 1, /* JOIN => ID */ 1, /* METRICS => ID */ - 1, /* STABLE => ID */ 1, /* INSERT => ID */ 1, /* INTO => ID */ 1, /* VALUES => ID */ @@ -685,6 +728,7 @@ struct yyParser { int yyerrcnt; /* Shifts left before out of the error */ #endif ParseARG_SDECL /* A place to hold %extra_argument */ + ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ @@ -799,212 +843,220 @@ static const char *const yyTokenName[] = { /* 64 */ "STABLES", /* 65 */ "VGROUPS", /* 66 */ "DROP", - /* 67 */ "DNODE", - /* 68 */ "USER", - /* 69 */ "ACCOUNT", - /* 70 */ "USE", - /* 71 */ "DESCRIBE", - /* 72 */ "ALTER", - /* 73 */ "PASS", - /* 74 */ "PRIVILEGE", - /* 75 */ "LOCAL", - /* 76 */ "IF", - /* 77 */ "EXISTS", - /* 78 */ "PPS", - /* 79 */ "TSERIES", - /* 80 */ "DBS", - /* 81 */ "STORAGE", - /* 82 */ "QTIME", - /* 83 */ "CONNS", - /* 84 */ "STATE", - /* 85 */ "KEEP", - /* 86 */ "CACHE", - /* 87 */ "REPLICA", - /* 88 */ "QUORUM", - /* 89 */ "DAYS", - /* 90 */ "MINROWS", - /* 91 */ "MAXROWS", - /* 92 */ "BLOCKS", - /* 93 */ "CTIME", - /* 94 */ "WAL", - /* 95 */ "FSYNC", - /* 96 */ "COMP", - /* 97 */ "PRECISION", - /* 98 */ "UPDATE", - /* 99 */ "LP", - /* 100 */ "RP", - /* 101 */ "TAGS", - /* 102 */ "USING", - /* 103 */ "AS", - /* 104 */ "COMMA", - /* 105 */ "NULL", - /* 106 */ "SELECT", - /* 107 */ "UNION", - /* 108 */ "ALL", - /* 109 */ "FROM", - /* 110 */ "VARIABLE", - /* 111 */ "INTERVAL", - /* 112 */ "FILL", - /* 113 */ "SLIDING", - /* 114 */ "ORDER", - /* 115 */ "BY", - /* 116 */ "ASC", - /* 117 */ "DESC", - /* 118 */ "GROUP", - /* 119 */ "HAVING", - /* 120 */ "LIMIT", - /* 121 */ "OFFSET", - /* 122 */ "SLIMIT", - /* 123 */ "SOFFSET", - /* 124 */ "WHERE", - /* 125 */ "NOW", - /* 126 */ "RESET", - /* 127 */ "QUERY", - /* 128 */ "ADD", - /* 129 */ "COLUMN", - /* 130 */ "TAG", - /* 131 */ "CHANGE", - /* 132 */ "SET", - /* 133 */ "KILL", - /* 134 */ "CONNECTION", - /* 135 */ "STREAM", - /* 136 */ "COLON", - /* 137 */ "ABORT", - /* 138 */ "AFTER", - /* 139 */ "ATTACH", - /* 140 */ "BEFORE", - /* 141 */ "BEGIN", - /* 142 */ "CASCADE", - /* 143 */ "CLUSTER", - /* 144 */ "CONFLICT", - /* 145 */ "COPY", - /* 146 */ "DEFERRED", - /* 147 */ "DELIMITERS", - /* 148 */ "DETACH", - /* 149 */ "EACH", - /* 150 */ "END", - /* 151 */ "EXPLAIN", - /* 152 */ "FAIL", - /* 153 */ "FOR", - /* 154 */ "IGNORE", - /* 155 */ "IMMEDIATE", - /* 156 */ "INITIALLY", - /* 157 */ "INSTEAD", - /* 158 */ "MATCH", - /* 159 */ "KEY", - /* 160 */ "OF", - /* 161 */ "RAISE", - /* 162 */ "REPLACE", - /* 163 */ "RESTRICT", - /* 164 */ "ROW", - /* 165 */ "STATEMENT", - /* 166 */ "TRIGGER", - /* 167 */ "VIEW", - /* 168 */ "COUNT", - /* 169 */ "SUM", - /* 170 */ "AVG", - /* 171 */ "MIN", - /* 172 */ "MAX", - /* 173 */ "FIRST", - /* 174 */ "LAST", - /* 175 */ "TOP", - /* 176 */ "BOTTOM", - /* 177 */ "STDDEV", - /* 178 */ "PERCENTILE", - /* 179 */ "APERCENTILE", - /* 180 */ "LEASTSQUARES", - /* 181 */ "HISTOGRAM", - /* 182 */ "DIFF", - /* 183 */ "SPREAD", - /* 184 */ "TWA", - /* 185 */ "INTERP", - /* 186 */ "LAST_ROW", - /* 187 */ "RATE", - /* 188 */ "IRATE", - /* 189 */ "SUM_RATE", - /* 190 */ "SUM_IRATE", - /* 191 */ "AVG_RATE", - /* 192 */ "AVG_IRATE", - /* 193 */ "TBID", - /* 194 */ "SEMI", - /* 195 */ "NONE", - /* 196 */ "PREV", - /* 197 */ "LINEAR", - /* 198 */ "IMPORT", - /* 199 */ "METRIC", - /* 200 */ "TBNAME", - /* 201 */ "JOIN", - /* 202 */ "METRICS", - /* 203 */ "STABLE", - /* 204 */ "INSERT", - /* 205 */ "INTO", - /* 206 */ "VALUES", - /* 207 */ "error", - /* 208 */ "program", - /* 209 */ "cmd", - /* 210 */ "dbPrefix", - /* 211 */ "ids", - /* 212 */ "cpxName", - /* 213 */ "ifexists", - /* 214 */ "alter_db_optr", - /* 215 */ "acct_optr", - /* 216 */ "ifnotexists", - /* 217 */ "db_optr", - /* 218 */ "pps", - /* 219 */ "tseries", - /* 220 */ "dbs", - /* 221 */ "streams", - /* 222 */ "storage", - /* 223 */ "qtime", - /* 224 */ "users", - /* 225 */ "conns", - /* 226 */ "state", - /* 227 */ "keep", - /* 228 */ "tagitemlist", - /* 229 */ "cache", - /* 230 */ "replica", - /* 231 */ "quorum", - /* 232 */ "days", - /* 233 */ "minrows", - /* 234 */ "maxrows", - /* 235 */ "blocks", - /* 236 */ "ctime", - /* 237 */ "wal", - /* 238 */ "fsync", - /* 239 */ "comp", - /* 240 */ "prec", - /* 241 */ "update", - /* 242 */ "typename", - /* 243 */ "signed", - /* 244 */ "create_table_args", - /* 245 */ "columnlist", - /* 246 */ "select", - /* 247 */ "column", - /* 248 */ "tagitem", - /* 249 */ "selcollist", - /* 250 */ "from", - /* 251 */ "where_opt", - /* 252 */ "interval_opt", - /* 253 */ "fill_opt", - /* 254 */ "sliding_opt", - /* 255 */ "groupby_opt", - /* 256 */ "orderby_opt", - /* 257 */ "having_opt", - /* 258 */ "slimit_opt", - /* 259 */ "limit_opt", - /* 260 */ "union", - /* 261 */ "sclp", - /* 262 */ "expr", - /* 263 */ "as", - /* 264 */ "tablelist", - /* 265 */ "tmvar", - /* 266 */ "sortlist", - /* 267 */ "sortitem", - /* 268 */ "item", - /* 269 */ "sortorder", - /* 270 */ "grouplist", - /* 271 */ "exprlist", - /* 272 */ "expritem", + /* 67 */ "STABLE", + /* 68 */ "DNODE", + /* 69 */ "USER", + /* 70 */ "ACCOUNT", + /* 71 */ "USE", + /* 72 */ "DESCRIBE", + /* 73 */ "ALTER", + /* 74 */ "PASS", + /* 75 */ "PRIVILEGE", + /* 76 */ "LOCAL", + /* 77 */ "IF", + /* 78 */ "EXISTS", + /* 79 */ "PPS", + /* 80 */ "TSERIES", + /* 81 */ "DBS", + /* 82 */ "STORAGE", + /* 83 */ "QTIME", + /* 84 */ "CONNS", + /* 85 */ "STATE", + /* 86 */ "KEEP", + /* 87 */ "CACHE", + /* 88 */ "REPLICA", + /* 89 */ "QUORUM", + /* 90 */ "DAYS", + /* 91 */ "MINROWS", + /* 92 */ "MAXROWS", + /* 93 */ "BLOCKS", + /* 94 */ "CTIME", + /* 95 */ "WAL", + /* 96 */ "FSYNC", + /* 97 */ "COMP", + /* 98 */ "PRECISION", + /* 99 */ "UPDATE", + /* 100 */ "CACHELAST", + /* 101 */ "LP", + /* 102 */ "RP", + /* 103 */ "UNSIGNED", + /* 104 */ "TAGS", + /* 105 */ "USING", + /* 106 */ "COMMA", + /* 107 */ "AS", + /* 108 */ "NULL", + /* 109 */ "SELECT", + /* 110 */ "UNION", + /* 111 */ "ALL", + /* 112 */ "DISTINCT", + /* 113 */ "FROM", + /* 114 */ "VARIABLE", + /* 115 */ "INTERVAL", + /* 116 */ "FILL", + /* 117 */ "SLIDING", + /* 118 */ "ORDER", + /* 119 */ "BY", + /* 120 */ "ASC", + /* 121 */ "DESC", + /* 122 */ "GROUP", + /* 123 */ "HAVING", + /* 124 */ "LIMIT", + /* 125 */ "OFFSET", + /* 126 */ "SLIMIT", + /* 127 */ "SOFFSET", + /* 128 */ "WHERE", + /* 129 */ "NOW", + /* 130 */ "RESET", + /* 131 */ "QUERY", + /* 132 */ "ADD", + /* 133 */ "COLUMN", + /* 134 */ "TAG", + /* 135 */ "CHANGE", + /* 136 */ "SET", + /* 137 */ "KILL", + /* 138 */ "CONNECTION", + /* 139 */ "STREAM", + /* 140 */ "COLON", + /* 141 */ "ABORT", + /* 142 */ "AFTER", + /* 143 */ "ATTACH", + /* 144 */ "BEFORE", + /* 145 */ "BEGIN", + /* 146 */ "CASCADE", + /* 147 */ "CLUSTER", + /* 148 */ "CONFLICT", + /* 149 */ "COPY", + /* 150 */ "DEFERRED", + /* 151 */ "DELIMITERS", + /* 152 */ "DETACH", + /* 153 */ "EACH", + /* 154 */ "END", + /* 155 */ "EXPLAIN", + /* 156 */ "FAIL", + /* 157 */ "FOR", + /* 158 */ "IGNORE", + /* 159 */ "IMMEDIATE", + /* 160 */ "INITIALLY", + /* 161 */ "INSTEAD", + /* 162 */ "MATCH", + /* 163 */ "KEY", + /* 164 */ "OF", + /* 165 */ "RAISE", + /* 166 */ "REPLACE", + /* 167 */ "RESTRICT", + /* 168 */ "ROW", + /* 169 */ "STATEMENT", + /* 170 */ "TRIGGER", + /* 171 */ "VIEW", + /* 172 */ "COUNT", + /* 173 */ "SUM", + /* 174 */ "AVG", + /* 175 */ "MIN", + /* 176 */ "MAX", + /* 177 */ "FIRST", + /* 178 */ "LAST", + /* 179 */ "TOP", + /* 180 */ "BOTTOM", + /* 181 */ "STDDEV", + /* 182 */ "PERCENTILE", + /* 183 */ "APERCENTILE", + /* 184 */ "LEASTSQUARES", + /* 185 */ "HISTOGRAM", + /* 186 */ "DIFF", + /* 187 */ "SPREAD", + /* 188 */ "TWA", + /* 189 */ "INTERP", + /* 190 */ "LAST_ROW", + /* 191 */ "RATE", + /* 192 */ "IRATE", + /* 193 */ "SUM_RATE", + /* 194 */ "SUM_IRATE", + /* 195 */ "AVG_RATE", + /* 196 */ "AVG_IRATE", + /* 197 */ "TBID", + /* 198 */ "SEMI", + /* 199 */ "NONE", + /* 200 */ "PREV", + /* 201 */ "LINEAR", + /* 202 */ "IMPORT", + /* 203 */ "METRIC", + /* 204 */ "TBNAME", + /* 205 */ "JOIN", + /* 206 */ "METRICS", + /* 207 */ "INSERT", + /* 208 */ "INTO", + /* 209 */ "VALUES", + /* 210 */ "program", + /* 211 */ "cmd", + /* 212 */ "dbPrefix", + /* 213 */ "ids", + /* 214 */ "cpxName", + /* 215 */ "ifexists", + /* 216 */ "alter_db_optr", + /* 217 */ "acct_optr", + /* 218 */ "ifnotexists", + /* 219 */ "db_optr", + /* 220 */ "pps", + /* 221 */ "tseries", + /* 222 */ "dbs", + /* 223 */ "streams", + /* 224 */ "storage", + /* 225 */ "qtime", + /* 226 */ "users", + /* 227 */ "conns", + /* 228 */ "state", + /* 229 */ "keep", + /* 230 */ "tagitemlist", + /* 231 */ "cache", + /* 232 */ "replica", + /* 233 */ "quorum", + /* 234 */ "days", + /* 235 */ "minrows", + /* 236 */ "maxrows", + /* 237 */ "blocks", + /* 238 */ "ctime", + /* 239 */ "wal", + /* 240 */ "fsync", + /* 241 */ "comp", + /* 242 */ "prec", + /* 243 */ "update", + /* 244 */ "cachelast", + /* 245 */ "typename", + /* 246 */ "signed", + /* 247 */ "create_table_args", + /* 248 */ "create_stable_args", + /* 249 */ "create_table_list", + /* 250 */ "create_from_stable", + /* 251 */ "columnlist", + /* 252 */ "tagNamelist", + /* 253 */ "select", + /* 254 */ "column", + /* 255 */ "tagitem", + /* 256 */ "selcollist", + /* 257 */ "from", + /* 258 */ "where_opt", + /* 259 */ "interval_opt", + /* 260 */ "fill_opt", + /* 261 */ "sliding_opt", + /* 262 */ "groupby_opt", + /* 263 */ "orderby_opt", + /* 264 */ "having_opt", + /* 265 */ "slimit_opt", + /* 266 */ "limit_opt", + /* 267 */ "union", + /* 268 */ "sclp", + /* 269 */ "distinct", + /* 270 */ "expr", + /* 271 */ "as", + /* 272 */ "tablelist", + /* 273 */ "tmvar", + /* 274 */ "sortlist", + /* 275 */ "sortitem", + /* 276 */ "item", + /* 277 */ "sortorder", + /* 278 */ "grouplist", + /* 279 */ "exprlist", + /* 280 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1040,211 +1092,232 @@ static const char *const yyRuleName[] = { /* 25 */ "cmd ::= SHOW dbPrefix VGROUPS", /* 26 */ "cmd ::= SHOW dbPrefix VGROUPS ids", /* 27 */ "cmd ::= DROP TABLE ifexists ids cpxName", - /* 28 */ "cmd ::= DROP DATABASE ifexists ids", - /* 29 */ "cmd ::= DROP DNODE ids", - /* 30 */ "cmd ::= DROP USER ids", - /* 31 */ "cmd ::= DROP ACCOUNT ids", - /* 32 */ "cmd ::= USE ids", - /* 33 */ "cmd ::= DESCRIBE ids cpxName", - /* 34 */ "cmd ::= ALTER USER ids PASS ids", - /* 35 */ "cmd ::= ALTER USER ids PRIVILEGE ids", - /* 36 */ "cmd ::= ALTER DNODE ids ids", - /* 37 */ "cmd ::= ALTER DNODE ids ids ids", - /* 38 */ "cmd ::= ALTER LOCAL ids", - /* 39 */ "cmd ::= ALTER LOCAL ids ids", - /* 40 */ "cmd ::= ALTER DATABASE ids alter_db_optr", - /* 41 */ "cmd ::= ALTER ACCOUNT ids acct_optr", - /* 42 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", - /* 43 */ "ids ::= ID", - /* 44 */ "ids ::= STRING", - /* 45 */ "ifexists ::= IF EXISTS", - /* 46 */ "ifexists ::=", - /* 47 */ "ifnotexists ::= IF NOT EXISTS", - /* 48 */ "ifnotexists ::=", - /* 49 */ "cmd ::= CREATE DNODE ids", - /* 50 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", - /* 51 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", - /* 52 */ "cmd ::= CREATE USER ids PASS ids", - /* 53 */ "pps ::=", - /* 54 */ "pps ::= PPS INTEGER", - /* 55 */ "tseries ::=", - /* 56 */ "tseries ::= TSERIES INTEGER", - /* 57 */ "dbs ::=", - /* 58 */ "dbs ::= DBS INTEGER", - /* 59 */ "streams ::=", - /* 60 */ "streams ::= STREAMS INTEGER", - /* 61 */ "storage ::=", - /* 62 */ "storage ::= STORAGE INTEGER", - /* 63 */ "qtime ::=", - /* 64 */ "qtime ::= QTIME INTEGER", - /* 65 */ "users ::=", - /* 66 */ "users ::= USERS INTEGER", - /* 67 */ "conns ::=", - /* 68 */ "conns ::= CONNS INTEGER", - /* 69 */ "state ::=", - /* 70 */ "state ::= STATE ids", - /* 71 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", - /* 72 */ "keep ::= KEEP tagitemlist", - /* 73 */ "cache ::= CACHE INTEGER", - /* 74 */ "replica ::= REPLICA INTEGER", - /* 75 */ "quorum ::= QUORUM INTEGER", - /* 76 */ "days ::= DAYS INTEGER", - /* 77 */ "minrows ::= MINROWS INTEGER", - /* 78 */ "maxrows ::= MAXROWS INTEGER", - /* 79 */ "blocks ::= BLOCKS INTEGER", - /* 80 */ "ctime ::= CTIME INTEGER", - /* 81 */ "wal ::= WAL INTEGER", - /* 82 */ "fsync ::= FSYNC INTEGER", - /* 83 */ "comp ::= COMP INTEGER", - /* 84 */ "prec ::= PRECISION STRING", - /* 85 */ "update ::= UPDATE INTEGER", - /* 86 */ "db_optr ::=", - /* 87 */ "db_optr ::= db_optr cache", - /* 88 */ "db_optr ::= db_optr replica", - /* 89 */ "db_optr ::= db_optr quorum", - /* 90 */ "db_optr ::= db_optr days", - /* 91 */ "db_optr ::= db_optr minrows", - /* 92 */ "db_optr ::= db_optr maxrows", - /* 93 */ "db_optr ::= db_optr blocks", - /* 94 */ "db_optr ::= db_optr ctime", - /* 95 */ "db_optr ::= db_optr wal", - /* 96 */ "db_optr ::= db_optr fsync", - /* 97 */ "db_optr ::= db_optr comp", - /* 98 */ "db_optr ::= db_optr prec", - /* 99 */ "db_optr ::= db_optr keep", - /* 100 */ "db_optr ::= db_optr update", - /* 101 */ "alter_db_optr ::=", - /* 102 */ "alter_db_optr ::= alter_db_optr replica", - /* 103 */ "alter_db_optr ::= alter_db_optr quorum", - /* 104 */ "alter_db_optr ::= alter_db_optr keep", - /* 105 */ "alter_db_optr ::= alter_db_optr blocks", - /* 106 */ "alter_db_optr ::= alter_db_optr comp", - /* 107 */ "alter_db_optr ::= alter_db_optr wal", - /* 108 */ "alter_db_optr ::= alter_db_optr fsync", - /* 109 */ "alter_db_optr ::= alter_db_optr update", - /* 110 */ "typename ::= ids", - /* 111 */ "typename ::= ids LP signed RP", - /* 112 */ "signed ::= INTEGER", - /* 113 */ "signed ::= PLUS INTEGER", - /* 114 */ "signed ::= MINUS INTEGER", - /* 115 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", - /* 116 */ "create_table_args ::= LP columnlist RP", - /* 117 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", - /* 118 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", - /* 119 */ "create_table_args ::= AS select", - /* 120 */ "columnlist ::= columnlist COMMA column", - /* 121 */ "columnlist ::= column", - /* 122 */ "column ::= ids typename", - /* 123 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 124 */ "tagitemlist ::= tagitem", - /* 125 */ "tagitem ::= INTEGER", - /* 126 */ "tagitem ::= FLOAT", - /* 127 */ "tagitem ::= STRING", - /* 128 */ "tagitem ::= BOOL", - /* 129 */ "tagitem ::= NULL", - /* 130 */ "tagitem ::= MINUS INTEGER", - /* 131 */ "tagitem ::= MINUS FLOAT", - /* 132 */ "tagitem ::= PLUS INTEGER", - /* 133 */ "tagitem ::= PLUS FLOAT", - /* 134 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 135 */ "union ::= select", - /* 136 */ "union ::= LP union RP", - /* 137 */ "union ::= union UNION ALL select", - /* 138 */ "union ::= union UNION ALL LP select RP", - /* 139 */ "cmd ::= union", - /* 140 */ "select ::= SELECT selcollist", - /* 141 */ "sclp ::= selcollist COMMA", - /* 142 */ "sclp ::=", - /* 143 */ "selcollist ::= sclp expr as", - /* 144 */ "selcollist ::= sclp STAR", - /* 145 */ "as ::= AS ids", - /* 146 */ "as ::= ids", - /* 147 */ "as ::=", - /* 148 */ "from ::= FROM tablelist", - /* 149 */ "tablelist ::= ids cpxName", - /* 150 */ "tablelist ::= ids cpxName ids", - /* 151 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 152 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 153 */ "tmvar ::= VARIABLE", - /* 154 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 155 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 156 */ "interval_opt ::=", - /* 157 */ "fill_opt ::=", - /* 158 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 159 */ "fill_opt ::= FILL LP ID RP", - /* 160 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 161 */ "sliding_opt ::=", - /* 162 */ "orderby_opt ::=", - /* 163 */ "orderby_opt ::= ORDER BY sortlist", - /* 164 */ "sortlist ::= sortlist COMMA item sortorder", - /* 165 */ "sortlist ::= item sortorder", - /* 166 */ "item ::= ids cpxName", - /* 167 */ "sortorder ::= ASC", - /* 168 */ "sortorder ::= DESC", - /* 169 */ "sortorder ::=", - /* 170 */ "groupby_opt ::=", - /* 171 */ "groupby_opt ::= GROUP BY grouplist", - /* 172 */ "grouplist ::= grouplist COMMA item", - /* 173 */ "grouplist ::= item", - /* 174 */ "having_opt ::=", - /* 175 */ "having_opt ::= HAVING expr", - /* 176 */ "limit_opt ::=", - /* 177 */ "limit_opt ::= LIMIT signed", - /* 178 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 179 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 180 */ "slimit_opt ::=", - /* 181 */ "slimit_opt ::= SLIMIT signed", - /* 182 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 183 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 184 */ "where_opt ::=", - /* 185 */ "where_opt ::= WHERE expr", - /* 186 */ "expr ::= LP expr RP", - /* 187 */ "expr ::= ID", - /* 188 */ "expr ::= ID DOT ID", - /* 189 */ "expr ::= ID DOT STAR", - /* 190 */ "expr ::= INTEGER", - /* 191 */ "expr ::= MINUS INTEGER", - /* 192 */ "expr ::= PLUS INTEGER", - /* 193 */ "expr ::= FLOAT", - /* 194 */ "expr ::= MINUS FLOAT", - /* 195 */ "expr ::= PLUS FLOAT", - /* 196 */ "expr ::= STRING", - /* 197 */ "expr ::= NOW", - /* 198 */ "expr ::= VARIABLE", - /* 199 */ "expr ::= BOOL", - /* 200 */ "expr ::= ID LP exprlist RP", - /* 201 */ "expr ::= ID LP STAR RP", - /* 202 */ "expr ::= expr IS NULL", - /* 203 */ "expr ::= expr IS NOT NULL", - /* 204 */ "expr ::= expr LT expr", - /* 205 */ "expr ::= expr GT expr", - /* 206 */ "expr ::= expr LE expr", - /* 207 */ "expr ::= expr GE expr", - /* 208 */ "expr ::= expr NE expr", - /* 209 */ "expr ::= expr EQ expr", - /* 210 */ "expr ::= expr AND expr", - /* 211 */ "expr ::= expr OR expr", - /* 212 */ "expr ::= expr PLUS expr", - /* 213 */ "expr ::= expr MINUS expr", - /* 214 */ "expr ::= expr STAR expr", - /* 215 */ "expr ::= expr SLASH expr", - /* 216 */ "expr ::= expr REM expr", - /* 217 */ "expr ::= expr LIKE expr", - /* 218 */ "expr ::= expr IN LP exprlist RP", - /* 219 */ "exprlist ::= exprlist COMMA expritem", - /* 220 */ "exprlist ::= expritem", - /* 221 */ "expritem ::= expr", - /* 222 */ "expritem ::=", - /* 223 */ "cmd ::= RESET QUERY CACHE", - /* 224 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 225 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 226 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 227 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 228 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 229 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 230 */ "cmd ::= KILL CONNECTION INTEGER", - /* 231 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 232 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 28 */ "cmd ::= DROP STABLE ifexists ids cpxName", + /* 29 */ "cmd ::= DROP DATABASE ifexists ids", + /* 30 */ "cmd ::= DROP DNODE ids", + /* 31 */ "cmd ::= DROP USER ids", + /* 32 */ "cmd ::= DROP ACCOUNT ids", + /* 33 */ "cmd ::= USE ids", + /* 34 */ "cmd ::= DESCRIBE ids cpxName", + /* 35 */ "cmd ::= ALTER USER ids PASS ids", + /* 36 */ "cmd ::= ALTER USER ids PRIVILEGE ids", + /* 37 */ "cmd ::= ALTER DNODE ids ids", + /* 38 */ "cmd ::= ALTER DNODE ids ids ids", + /* 39 */ "cmd ::= ALTER LOCAL ids", + /* 40 */ "cmd ::= ALTER LOCAL ids ids", + /* 41 */ "cmd ::= ALTER DATABASE ids alter_db_optr", + /* 42 */ "cmd ::= ALTER ACCOUNT ids acct_optr", + /* 43 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", + /* 44 */ "ids ::= ID", + /* 45 */ "ids ::= STRING", + /* 46 */ "ifexists ::= IF EXISTS", + /* 47 */ "ifexists ::=", + /* 48 */ "ifnotexists ::= IF NOT EXISTS", + /* 49 */ "ifnotexists ::=", + /* 50 */ "cmd ::= CREATE DNODE ids", + /* 51 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", + /* 52 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", + /* 53 */ "cmd ::= CREATE USER ids PASS ids", + /* 54 */ "pps ::=", + /* 55 */ "pps ::= PPS INTEGER", + /* 56 */ "tseries ::=", + /* 57 */ "tseries ::= TSERIES INTEGER", + /* 58 */ "dbs ::=", + /* 59 */ "dbs ::= DBS INTEGER", + /* 60 */ "streams ::=", + /* 61 */ "streams ::= STREAMS INTEGER", + /* 62 */ "storage ::=", + /* 63 */ "storage ::= STORAGE INTEGER", + /* 64 */ "qtime ::=", + /* 65 */ "qtime ::= QTIME INTEGER", + /* 66 */ "users ::=", + /* 67 */ "users ::= USERS INTEGER", + /* 68 */ "conns ::=", + /* 69 */ "conns ::= CONNS INTEGER", + /* 70 */ "state ::=", + /* 71 */ "state ::= STATE ids", + /* 72 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", + /* 73 */ "keep ::= KEEP tagitemlist", + /* 74 */ "cache ::= CACHE INTEGER", + /* 75 */ "replica ::= REPLICA INTEGER", + /* 76 */ "quorum ::= QUORUM INTEGER", + /* 77 */ "days ::= DAYS INTEGER", + /* 78 */ "minrows ::= MINROWS INTEGER", + /* 79 */ "maxrows ::= MAXROWS INTEGER", + /* 80 */ "blocks ::= BLOCKS INTEGER", + /* 81 */ "ctime ::= CTIME INTEGER", + /* 82 */ "wal ::= WAL INTEGER", + /* 83 */ "fsync ::= FSYNC INTEGER", + /* 84 */ "comp ::= COMP INTEGER", + /* 85 */ "prec ::= PRECISION STRING", + /* 86 */ "update ::= UPDATE INTEGER", + /* 87 */ "cachelast ::= CACHELAST INTEGER", + /* 88 */ "db_optr ::=", + /* 89 */ "db_optr ::= db_optr cache", + /* 90 */ "db_optr ::= db_optr replica", + /* 91 */ "db_optr ::= db_optr quorum", + /* 92 */ "db_optr ::= db_optr days", + /* 93 */ "db_optr ::= db_optr minrows", + /* 94 */ "db_optr ::= db_optr maxrows", + /* 95 */ "db_optr ::= db_optr blocks", + /* 96 */ "db_optr ::= db_optr ctime", + /* 97 */ "db_optr ::= db_optr wal", + /* 98 */ "db_optr ::= db_optr fsync", + /* 99 */ "db_optr ::= db_optr comp", + /* 100 */ "db_optr ::= db_optr prec", + /* 101 */ "db_optr ::= db_optr keep", + /* 102 */ "db_optr ::= db_optr update", + /* 103 */ "db_optr ::= db_optr cachelast", + /* 104 */ "alter_db_optr ::=", + /* 105 */ "alter_db_optr ::= alter_db_optr replica", + /* 106 */ "alter_db_optr ::= alter_db_optr quorum", + /* 107 */ "alter_db_optr ::= alter_db_optr keep", + /* 108 */ "alter_db_optr ::= alter_db_optr blocks", + /* 109 */ "alter_db_optr ::= alter_db_optr comp", + /* 110 */ "alter_db_optr ::= alter_db_optr wal", + /* 111 */ "alter_db_optr ::= alter_db_optr fsync", + /* 112 */ "alter_db_optr ::= alter_db_optr update", + /* 113 */ "alter_db_optr ::= alter_db_optr cachelast", + /* 114 */ "typename ::= ids", + /* 115 */ "typename ::= ids LP signed RP", + /* 116 */ "typename ::= ids UNSIGNED", + /* 117 */ "signed ::= INTEGER", + /* 118 */ "signed ::= PLUS INTEGER", + /* 119 */ "signed ::= MINUS INTEGER", + /* 120 */ "cmd ::= CREATE TABLE create_table_args", + /* 121 */ "cmd ::= CREATE TABLE create_stable_args", + /* 122 */ "cmd ::= CREATE STABLE create_stable_args", + /* 123 */ "cmd ::= CREATE TABLE create_table_list", + /* 124 */ "create_table_list ::= create_from_stable", + /* 125 */ "create_table_list ::= create_table_list create_from_stable", + /* 126 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", + /* 127 */ "create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", + /* 128 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", + /* 129 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP", + /* 130 */ "tagNamelist ::= tagNamelist COMMA ids", + /* 131 */ "tagNamelist ::= ids", + /* 132 */ "create_table_args ::= ifnotexists ids cpxName AS select", + /* 133 */ "columnlist ::= columnlist COMMA column", + /* 134 */ "columnlist ::= column", + /* 135 */ "column ::= ids typename", + /* 136 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 137 */ "tagitemlist ::= tagitem", + /* 138 */ "tagitem ::= INTEGER", + /* 139 */ "tagitem ::= FLOAT", + /* 140 */ "tagitem ::= STRING", + /* 141 */ "tagitem ::= BOOL", + /* 142 */ "tagitem ::= NULL", + /* 143 */ "tagitem ::= MINUS INTEGER", + /* 144 */ "tagitem ::= MINUS FLOAT", + /* 145 */ "tagitem ::= PLUS INTEGER", + /* 146 */ "tagitem ::= PLUS FLOAT", + /* 147 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 148 */ "union ::= select", + /* 149 */ "union ::= LP union RP", + /* 150 */ "union ::= union UNION ALL select", + /* 151 */ "union ::= union UNION ALL LP select RP", + /* 152 */ "cmd ::= union", + /* 153 */ "select ::= SELECT selcollist", + /* 154 */ "sclp ::= selcollist COMMA", + /* 155 */ "sclp ::=", + /* 156 */ "selcollist ::= sclp distinct expr as", + /* 157 */ "selcollist ::= sclp STAR", + /* 158 */ "as ::= AS ids", + /* 159 */ "as ::= ids", + /* 160 */ "as ::=", + /* 161 */ "distinct ::= DISTINCT", + /* 162 */ "distinct ::=", + /* 163 */ "from ::= FROM tablelist", + /* 164 */ "tablelist ::= ids cpxName", + /* 165 */ "tablelist ::= ids cpxName ids", + /* 166 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 167 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 168 */ "tmvar ::= VARIABLE", + /* 169 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 170 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 171 */ "interval_opt ::=", + /* 172 */ "fill_opt ::=", + /* 173 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 174 */ "fill_opt ::= FILL LP ID RP", + /* 175 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 176 */ "sliding_opt ::=", + /* 177 */ "orderby_opt ::=", + /* 178 */ "orderby_opt ::= ORDER BY sortlist", + /* 179 */ "sortlist ::= sortlist COMMA item sortorder", + /* 180 */ "sortlist ::= item sortorder", + /* 181 */ "item ::= ids cpxName", + /* 182 */ "sortorder ::= ASC", + /* 183 */ "sortorder ::= DESC", + /* 184 */ "sortorder ::=", + /* 185 */ "groupby_opt ::=", + /* 186 */ "groupby_opt ::= GROUP BY grouplist", + /* 187 */ "grouplist ::= grouplist COMMA item", + /* 188 */ "grouplist ::= item", + /* 189 */ "having_opt ::=", + /* 190 */ "having_opt ::= HAVING expr", + /* 191 */ "limit_opt ::=", + /* 192 */ "limit_opt ::= LIMIT signed", + /* 193 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 194 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 195 */ "slimit_opt ::=", + /* 196 */ "slimit_opt ::= SLIMIT signed", + /* 197 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 198 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 199 */ "where_opt ::=", + /* 200 */ "where_opt ::= WHERE expr", + /* 201 */ "expr ::= LP expr RP", + /* 202 */ "expr ::= ID", + /* 203 */ "expr ::= ID DOT ID", + /* 204 */ "expr ::= ID DOT STAR", + /* 205 */ "expr ::= INTEGER", + /* 206 */ "expr ::= MINUS INTEGER", + /* 207 */ "expr ::= PLUS INTEGER", + /* 208 */ "expr ::= FLOAT", + /* 209 */ "expr ::= MINUS FLOAT", + /* 210 */ "expr ::= PLUS FLOAT", + /* 211 */ "expr ::= STRING", + /* 212 */ "expr ::= NOW", + /* 213 */ "expr ::= VARIABLE", + /* 214 */ "expr ::= BOOL", + /* 215 */ "expr ::= ID LP exprlist RP", + /* 216 */ "expr ::= ID LP STAR RP", + /* 217 */ "expr ::= expr IS NULL", + /* 218 */ "expr ::= expr IS NOT NULL", + /* 219 */ "expr ::= expr LT expr", + /* 220 */ "expr ::= expr GT expr", + /* 221 */ "expr ::= expr LE expr", + /* 222 */ "expr ::= expr GE expr", + /* 223 */ "expr ::= expr NE expr", + /* 224 */ "expr ::= expr EQ expr", + /* 225 */ "expr ::= expr BETWEEN expr AND expr", + /* 226 */ "expr ::= expr AND expr", + /* 227 */ "expr ::= expr OR expr", + /* 228 */ "expr ::= expr PLUS expr", + /* 229 */ "expr ::= expr MINUS expr", + /* 230 */ "expr ::= expr STAR expr", + /* 231 */ "expr ::= expr SLASH expr", + /* 232 */ "expr ::= expr REM expr", + /* 233 */ "expr ::= expr LIKE expr", + /* 234 */ "expr ::= expr IN LP exprlist RP", + /* 235 */ "exprlist ::= exprlist COMMA expritem", + /* 236 */ "exprlist ::= expritem", + /* 237 */ "expritem ::= expr", + /* 238 */ "expritem ::=", + /* 239 */ "cmd ::= RESET QUERY CACHE", + /* 240 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 241 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 242 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 243 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 244 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 245 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 246 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", + /* 247 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", + /* 248 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", + /* 249 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", + /* 250 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", + /* 251 */ "cmd ::= KILL CONNECTION INTEGER", + /* 252 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 253 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1293,28 +1366,29 @@ static int yyGrowStack(yyParser *p){ /* Initialize a new parser that has already been allocated. */ -void ParseInit(void *yypParser){ - yyParser *pParser = (yyParser*)yypParser; +void ParseInit(void *yypRawParser ParseCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + ParseCTX_STORE #ifdef YYTRACKMAXSTACKDEPTH - pParser->yyhwm = 0; + yypParser->yyhwm = 0; #endif #if YYSTACKDEPTH<=0 - pParser->yytos = NULL; - pParser->yystack = NULL; - pParser->yystksz = 0; - if( yyGrowStack(pParser) ){ - pParser->yystack = &pParser->yystk0; - pParser->yystksz = 1; + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; } #endif #ifndef YYNOERRORRECOVERY - pParser->yyerrcnt = -1; + yypParser->yyerrcnt = -1; #endif - pParser->yytos = pParser->yystack; - pParser->yystack[0].stateno = 0; - pParser->yystack[0].major = 0; + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; #if YYSTACKDEPTH>0 - pParser->yystackEnd = &pParser->yystack[YYSTACKDEPTH-1]; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #endif } @@ -1331,11 +1405,14 @@ void ParseInit(void *yypParser){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ - yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( pParser ) ParseInit(pParser); - return pParser; +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + ParseCTX_STORE + ParseInit(yypParser ParseCTX_PARAM); + } + return (void*)yypParser; } #endif /* Parse_ENGINEALWAYSONSTACK */ @@ -1352,7 +1429,8 @@ static void yy_destructor( YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1365,46 +1443,52 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 227: /* keep */ - case 228: /* tagitemlist */ - case 245: /* columnlist */ - case 253: /* fill_opt */ - case 255: /* groupby_opt */ - case 256: /* orderby_opt */ - case 266: /* sortlist */ - case 270: /* grouplist */ + case 229: /* keep */ + case 230: /* tagitemlist */ + case 251: /* columnlist */ + case 252: /* tagNamelist */ + case 260: /* fill_opt */ + case 262: /* groupby_opt */ + case 263: /* orderby_opt */ + case 274: /* sortlist */ + case 278: /* grouplist */ +{ +taosArrayDestroy((yypminor->yy247)); +} + break; + case 249: /* create_table_list */ { -taosArrayDestroy((yypminor->yy165)); +destroyCreateTableSql((yypminor->yy358)); } break; - case 246: /* select */ + case 253: /* select */ { -doDestroyQuerySql((yypminor->yy414)); +doDestroyQuerySql((yypminor->yy114)); } break; - case 249: /* selcollist */ - case 261: /* sclp */ - case 271: /* exprlist */ + case 256: /* selcollist */ + case 268: /* sclp */ + case 279: /* exprlist */ { -tSQLExprListDestroy((yypminor->yy290)); +tSqlExprListDestroy((yypminor->yy522)); } break; - case 251: /* where_opt */ - case 257: /* having_opt */ - case 262: /* expr */ - case 272: /* expritem */ + case 258: /* where_opt */ + case 264: /* having_opt */ + case 270: /* expr */ + case 280: /* expritem */ { -tSQLExprDestroy((yypminor->yy64)); +tSqlExprDestroy((yypminor->yy326)); } break; - case 260: /* union */ + case 267: /* union */ { -destroyAllSelectClause((yypminor->yy231)); +destroyAllSelectClause((yypminor->yy219)); } break; - case 267: /* sortitem */ + case 275: /* sortitem */ { -tVariantDestroy(&(yypminor->yy134)); +tVariantDestroy(&(yypminor->yy378)); } break; /********* End destructor definitions *****************************************/ @@ -1516,13 +1600,12 @@ int ParseCoverage(FILE *out){ ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. */ -static unsigned int yy_find_shift_action( - yyParser *pParser, /* The parser */ - YYCODETYPE iLookAhead /* The look-ahead token */ +static YYACTIONTYPE yy_find_shift_action( + YYCODETYPE iLookAhead, /* The look-ahead token */ + YYACTIONTYPE stateno /* Current state number */ ){ int i; - int stateno = pParser->yytos->stateno; - + if( stateno>YY_MAX_SHIFT ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); #if defined(YYCOVERAGE) @@ -1530,15 +1613,19 @@ static unsigned int yy_find_shift_action( #endif do{ i = yy_shift_ofst[stateno]; - assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -1553,15 +1640,8 @@ static unsigned int yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -1575,6 +1655,7 @@ static unsigned int yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && iyytos; - yytos->stateno = (YYACTIONTYPE)yyNewState; - yytos->major = (YYCODETYPE)yyMajor; + yytos->stateno = yyNewState; + yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; yyTraceShift(yypParser, yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - signed char nrhs; /* Negative of the number of RHS symbols in the rule */ -} yyRuleInfo[] = { - { 208, -1 }, /* (0) program ::= cmd */ - { 209, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 209, -2 }, /* (2) cmd ::= SHOW MNODES */ - { 209, -2 }, /* (3) cmd ::= SHOW DNODES */ - { 209, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ - { 209, -2 }, /* (5) cmd ::= SHOW USERS */ - { 209, -2 }, /* (6) cmd ::= SHOW MODULES */ - { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ - { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ - { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ - { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ - { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ - { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ - { 209, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - { 210, 0 }, /* (15) dbPrefix ::= */ - { 210, -2 }, /* (16) dbPrefix ::= ids DOT */ - { 212, 0 }, /* (17) cpxName ::= */ - { 212, -2 }, /* (18) cpxName ::= DOT ids */ - { 209, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ - { 209, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ - { 209, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ - { 209, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 209, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ - { 209, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 209, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ - { 209, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 209, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ - { 209, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ - { 209, -3 }, /* (29) cmd ::= DROP DNODE ids */ - { 209, -3 }, /* (30) cmd ::= DROP USER ids */ - { 209, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ - { 209, -2 }, /* (32) cmd ::= USE ids */ - { 209, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ - { 209, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ - { 209, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 209, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ - { 209, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ - { 209, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ - { 209, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ - { 209, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 209, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 209, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 211, -1 }, /* (43) ids ::= ID */ - { 211, -1 }, /* (44) ids ::= STRING */ - { 213, -2 }, /* (45) ifexists ::= IF EXISTS */ - { 213, 0 }, /* (46) ifexists ::= */ - { 216, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ - { 216, 0 }, /* (48) ifnotexists ::= */ - { 209, -3 }, /* (49) cmd ::= CREATE DNODE ids */ - { 209, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 209, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 209, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ - { 218, 0 }, /* (53) pps ::= */ - { 218, -2 }, /* (54) pps ::= PPS INTEGER */ - { 219, 0 }, /* (55) tseries ::= */ - { 219, -2 }, /* (56) tseries ::= TSERIES INTEGER */ - { 220, 0 }, /* (57) dbs ::= */ - { 220, -2 }, /* (58) dbs ::= DBS INTEGER */ - { 221, 0 }, /* (59) streams ::= */ - { 221, -2 }, /* (60) streams ::= STREAMS INTEGER */ - { 222, 0 }, /* (61) storage ::= */ - { 222, -2 }, /* (62) storage ::= STORAGE INTEGER */ - { 223, 0 }, /* (63) qtime ::= */ - { 223, -2 }, /* (64) qtime ::= QTIME INTEGER */ - { 224, 0 }, /* (65) users ::= */ - { 224, -2 }, /* (66) users ::= USERS INTEGER */ - { 225, 0 }, /* (67) conns ::= */ - { 225, -2 }, /* (68) conns ::= CONNS INTEGER */ - { 226, 0 }, /* (69) state ::= */ - { 226, -2 }, /* (70) state ::= STATE ids */ - { 215, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 227, -2 }, /* (72) keep ::= KEEP tagitemlist */ - { 229, -2 }, /* (73) cache ::= CACHE INTEGER */ - { 230, -2 }, /* (74) replica ::= REPLICA INTEGER */ - { 231, -2 }, /* (75) quorum ::= QUORUM INTEGER */ - { 232, -2 }, /* (76) days ::= DAYS INTEGER */ - { 233, -2 }, /* (77) minrows ::= MINROWS INTEGER */ - { 234, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ - { 235, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ - { 236, -2 }, /* (80) ctime ::= CTIME INTEGER */ - { 237, -2 }, /* (81) wal ::= WAL INTEGER */ - { 238, -2 }, /* (82) fsync ::= FSYNC INTEGER */ - { 239, -2 }, /* (83) comp ::= COMP INTEGER */ - { 240, -2 }, /* (84) prec ::= PRECISION STRING */ - { 241, -2 }, /* (85) update ::= UPDATE INTEGER */ - { 217, 0 }, /* (86) db_optr ::= */ - { 217, -2 }, /* (87) db_optr ::= db_optr cache */ - { 217, -2 }, /* (88) db_optr ::= db_optr replica */ - { 217, -2 }, /* (89) db_optr ::= db_optr quorum */ - { 217, -2 }, /* (90) db_optr ::= db_optr days */ - { 217, -2 }, /* (91) db_optr ::= db_optr minrows */ - { 217, -2 }, /* (92) db_optr ::= db_optr maxrows */ - { 217, -2 }, /* (93) db_optr ::= db_optr blocks */ - { 217, -2 }, /* (94) db_optr ::= db_optr ctime */ - { 217, -2 }, /* (95) db_optr ::= db_optr wal */ - { 217, -2 }, /* (96) db_optr ::= db_optr fsync */ - { 217, -2 }, /* (97) db_optr ::= db_optr comp */ - { 217, -2 }, /* (98) db_optr ::= db_optr prec */ - { 217, -2 }, /* (99) db_optr ::= db_optr keep */ - { 217, -2 }, /* (100) db_optr ::= db_optr update */ - { 214, 0 }, /* (101) alter_db_optr ::= */ - { 214, -2 }, /* (102) alter_db_optr ::= alter_db_optr replica */ - { 214, -2 }, /* (103) alter_db_optr ::= alter_db_optr quorum */ - { 214, -2 }, /* (104) alter_db_optr ::= alter_db_optr keep */ - { 214, -2 }, /* (105) alter_db_optr ::= alter_db_optr blocks */ - { 214, -2 }, /* (106) alter_db_optr ::= alter_db_optr comp */ - { 214, -2 }, /* (107) alter_db_optr ::= alter_db_optr wal */ - { 214, -2 }, /* (108) alter_db_optr ::= alter_db_optr fsync */ - { 214, -2 }, /* (109) alter_db_optr ::= alter_db_optr update */ - { 242, -1 }, /* (110) typename ::= ids */ - { 242, -4 }, /* (111) typename ::= ids LP signed RP */ - { 243, -1 }, /* (112) signed ::= INTEGER */ - { 243, -2 }, /* (113) signed ::= PLUS INTEGER */ - { 243, -2 }, /* (114) signed ::= MINUS INTEGER */ - { 209, -6 }, /* (115) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - { 244, -3 }, /* (116) create_table_args ::= LP columnlist RP */ - { 244, -7 }, /* (117) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - { 244, -7 }, /* (118) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - { 244, -2 }, /* (119) create_table_args ::= AS select */ - { 245, -3 }, /* (120) columnlist ::= columnlist COMMA column */ - { 245, -1 }, /* (121) columnlist ::= column */ - { 247, -2 }, /* (122) column ::= ids typename */ - { 228, -3 }, /* (123) tagitemlist ::= tagitemlist COMMA tagitem */ - { 228, -1 }, /* (124) tagitemlist ::= tagitem */ - { 248, -1 }, /* (125) tagitem ::= INTEGER */ - { 248, -1 }, /* (126) tagitem ::= FLOAT */ - { 248, -1 }, /* (127) tagitem ::= STRING */ - { 248, -1 }, /* (128) tagitem ::= BOOL */ - { 248, -1 }, /* (129) tagitem ::= NULL */ - { 248, -2 }, /* (130) tagitem ::= MINUS INTEGER */ - { 248, -2 }, /* (131) tagitem ::= MINUS FLOAT */ - { 248, -2 }, /* (132) tagitem ::= PLUS INTEGER */ - { 248, -2 }, /* (133) tagitem ::= PLUS FLOAT */ - { 246, -12 }, /* (134) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - { 260, -1 }, /* (135) union ::= select */ - { 260, -3 }, /* (136) union ::= LP union RP */ - { 260, -4 }, /* (137) union ::= union UNION ALL select */ - { 260, -6 }, /* (138) union ::= union UNION ALL LP select RP */ - { 209, -1 }, /* (139) cmd ::= union */ - { 246, -2 }, /* (140) select ::= SELECT selcollist */ - { 261, -2 }, /* (141) sclp ::= selcollist COMMA */ - { 261, 0 }, /* (142) sclp ::= */ - { 249, -3 }, /* (143) selcollist ::= sclp expr as */ - { 249, -2 }, /* (144) selcollist ::= sclp STAR */ - { 263, -2 }, /* (145) as ::= AS ids */ - { 263, -1 }, /* (146) as ::= ids */ - { 263, 0 }, /* (147) as ::= */ - { 250, -2 }, /* (148) from ::= FROM tablelist */ - { 264, -2 }, /* (149) tablelist ::= ids cpxName */ - { 264, -3 }, /* (150) tablelist ::= ids cpxName ids */ - { 264, -4 }, /* (151) tablelist ::= tablelist COMMA ids cpxName */ - { 264, -5 }, /* (152) tablelist ::= tablelist COMMA ids cpxName ids */ - { 265, -1 }, /* (153) tmvar ::= VARIABLE */ - { 252, -4 }, /* (154) interval_opt ::= INTERVAL LP tmvar RP */ - { 252, -6 }, /* (155) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - { 252, 0 }, /* (156) interval_opt ::= */ - { 253, 0 }, /* (157) fill_opt ::= */ - { 253, -6 }, /* (158) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 253, -4 }, /* (159) fill_opt ::= FILL LP ID RP */ - { 254, -4 }, /* (160) sliding_opt ::= SLIDING LP tmvar RP */ - { 254, 0 }, /* (161) sliding_opt ::= */ - { 256, 0 }, /* (162) orderby_opt ::= */ - { 256, -3 }, /* (163) orderby_opt ::= ORDER BY sortlist */ - { 266, -4 }, /* (164) sortlist ::= sortlist COMMA item sortorder */ - { 266, -2 }, /* (165) sortlist ::= item sortorder */ - { 268, -2 }, /* (166) item ::= ids cpxName */ - { 269, -1 }, /* (167) sortorder ::= ASC */ - { 269, -1 }, /* (168) sortorder ::= DESC */ - { 269, 0 }, /* (169) sortorder ::= */ - { 255, 0 }, /* (170) groupby_opt ::= */ - { 255, -3 }, /* (171) groupby_opt ::= GROUP BY grouplist */ - { 270, -3 }, /* (172) grouplist ::= grouplist COMMA item */ - { 270, -1 }, /* (173) grouplist ::= item */ - { 257, 0 }, /* (174) having_opt ::= */ - { 257, -2 }, /* (175) having_opt ::= HAVING expr */ - { 259, 0 }, /* (176) limit_opt ::= */ - { 259, -2 }, /* (177) limit_opt ::= LIMIT signed */ - { 259, -4 }, /* (178) limit_opt ::= LIMIT signed OFFSET signed */ - { 259, -4 }, /* (179) limit_opt ::= LIMIT signed COMMA signed */ - { 258, 0 }, /* (180) slimit_opt ::= */ - { 258, -2 }, /* (181) slimit_opt ::= SLIMIT signed */ - { 258, -4 }, /* (182) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 258, -4 }, /* (183) slimit_opt ::= SLIMIT signed COMMA signed */ - { 251, 0 }, /* (184) where_opt ::= */ - { 251, -2 }, /* (185) where_opt ::= WHERE expr */ - { 262, -3 }, /* (186) expr ::= LP expr RP */ - { 262, -1 }, /* (187) expr ::= ID */ - { 262, -3 }, /* (188) expr ::= ID DOT ID */ - { 262, -3 }, /* (189) expr ::= ID DOT STAR */ - { 262, -1 }, /* (190) expr ::= INTEGER */ - { 262, -2 }, /* (191) expr ::= MINUS INTEGER */ - { 262, -2 }, /* (192) expr ::= PLUS INTEGER */ - { 262, -1 }, /* (193) expr ::= FLOAT */ - { 262, -2 }, /* (194) expr ::= MINUS FLOAT */ - { 262, -2 }, /* (195) expr ::= PLUS FLOAT */ - { 262, -1 }, /* (196) expr ::= STRING */ - { 262, -1 }, /* (197) expr ::= NOW */ - { 262, -1 }, /* (198) expr ::= VARIABLE */ - { 262, -1 }, /* (199) expr ::= BOOL */ - { 262, -4 }, /* (200) expr ::= ID LP exprlist RP */ - { 262, -4 }, /* (201) expr ::= ID LP STAR RP */ - { 262, -3 }, /* (202) expr ::= expr IS NULL */ - { 262, -4 }, /* (203) expr ::= expr IS NOT NULL */ - { 262, -3 }, /* (204) expr ::= expr LT expr */ - { 262, -3 }, /* (205) expr ::= expr GT expr */ - { 262, -3 }, /* (206) expr ::= expr LE expr */ - { 262, -3 }, /* (207) expr ::= expr GE expr */ - { 262, -3 }, /* (208) expr ::= expr NE expr */ - { 262, -3 }, /* (209) expr ::= expr EQ expr */ - { 262, -3 }, /* (210) expr ::= expr AND expr */ - { 262, -3 }, /* (211) expr ::= expr OR expr */ - { 262, -3 }, /* (212) expr ::= expr PLUS expr */ - { 262, -3 }, /* (213) expr ::= expr MINUS expr */ - { 262, -3 }, /* (214) expr ::= expr STAR expr */ - { 262, -3 }, /* (215) expr ::= expr SLASH expr */ - { 262, -3 }, /* (216) expr ::= expr REM expr */ - { 262, -3 }, /* (217) expr ::= expr LIKE expr */ - { 262, -5 }, /* (218) expr ::= expr IN LP exprlist RP */ - { 271, -3 }, /* (219) exprlist ::= exprlist COMMA expritem */ - { 271, -1 }, /* (220) exprlist ::= expritem */ - { 272, -1 }, /* (221) expritem ::= expr */ - { 272, 0 }, /* (222) expritem ::= */ - { 209, -3 }, /* (223) cmd ::= RESET QUERY CACHE */ - { 209, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 209, -7 }, /* (225) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 209, -7 }, /* (226) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 209, -7 }, /* (227) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 209, -8 }, /* (228) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 209, -9 }, /* (229) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 209, -3 }, /* (230) cmd ::= KILL CONNECTION INTEGER */ - { 209, -5 }, /* (231) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 209, -5 }, /* (232) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { + 210, /* (0) program ::= cmd */ + 211, /* (1) cmd ::= SHOW DATABASES */ + 211, /* (2) cmd ::= SHOW MNODES */ + 211, /* (3) cmd ::= SHOW DNODES */ + 211, /* (4) cmd ::= SHOW ACCOUNTS */ + 211, /* (5) cmd ::= SHOW USERS */ + 211, /* (6) cmd ::= SHOW MODULES */ + 211, /* (7) cmd ::= SHOW QUERIES */ + 211, /* (8) cmd ::= SHOW CONNECTIONS */ + 211, /* (9) cmd ::= SHOW STREAMS */ + 211, /* (10) cmd ::= SHOW VARIABLES */ + 211, /* (11) cmd ::= SHOW SCORES */ + 211, /* (12) cmd ::= SHOW GRANTS */ + 211, /* (13) cmd ::= SHOW VNODES */ + 211, /* (14) cmd ::= SHOW VNODES IPTOKEN */ + 212, /* (15) dbPrefix ::= */ + 212, /* (16) dbPrefix ::= ids DOT */ + 214, /* (17) cpxName ::= */ + 214, /* (18) cpxName ::= DOT ids */ + 211, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ + 211, /* (20) cmd ::= SHOW CREATE DATABASE ids */ + 211, /* (21) cmd ::= SHOW dbPrefix TABLES */ + 211, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + 211, /* (23) cmd ::= SHOW dbPrefix STABLES */ + 211, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + 211, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ + 211, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ + 211, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ + 211, /* (28) cmd ::= DROP STABLE ifexists ids cpxName */ + 211, /* (29) cmd ::= DROP DATABASE ifexists ids */ + 211, /* (30) cmd ::= DROP DNODE ids */ + 211, /* (31) cmd ::= DROP USER ids */ + 211, /* (32) cmd ::= DROP ACCOUNT ids */ + 211, /* (33) cmd ::= USE ids */ + 211, /* (34) cmd ::= DESCRIBE ids cpxName */ + 211, /* (35) cmd ::= ALTER USER ids PASS ids */ + 211, /* (36) cmd ::= ALTER USER ids PRIVILEGE ids */ + 211, /* (37) cmd ::= ALTER DNODE ids ids */ + 211, /* (38) cmd ::= ALTER DNODE ids ids ids */ + 211, /* (39) cmd ::= ALTER LOCAL ids */ + 211, /* (40) cmd ::= ALTER LOCAL ids ids */ + 211, /* (41) cmd ::= ALTER DATABASE ids alter_db_optr */ + 211, /* (42) cmd ::= ALTER ACCOUNT ids acct_optr */ + 211, /* (43) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + 213, /* (44) ids ::= ID */ + 213, /* (45) ids ::= STRING */ + 215, /* (46) ifexists ::= IF EXISTS */ + 215, /* (47) ifexists ::= */ + 218, /* (48) ifnotexists ::= IF NOT EXISTS */ + 218, /* (49) ifnotexists ::= */ + 211, /* (50) cmd ::= CREATE DNODE ids */ + 211, /* (51) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + 211, /* (52) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + 211, /* (53) cmd ::= CREATE USER ids PASS ids */ + 220, /* (54) pps ::= */ + 220, /* (55) pps ::= PPS INTEGER */ + 221, /* (56) tseries ::= */ + 221, /* (57) tseries ::= TSERIES INTEGER */ + 222, /* (58) dbs ::= */ + 222, /* (59) dbs ::= DBS INTEGER */ + 223, /* (60) streams ::= */ + 223, /* (61) streams ::= STREAMS INTEGER */ + 224, /* (62) storage ::= */ + 224, /* (63) storage ::= STORAGE INTEGER */ + 225, /* (64) qtime ::= */ + 225, /* (65) qtime ::= QTIME INTEGER */ + 226, /* (66) users ::= */ + 226, /* (67) users ::= USERS INTEGER */ + 227, /* (68) conns ::= */ + 227, /* (69) conns ::= CONNS INTEGER */ + 228, /* (70) state ::= */ + 228, /* (71) state ::= STATE ids */ + 217, /* (72) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + 229, /* (73) keep ::= KEEP tagitemlist */ + 231, /* (74) cache ::= CACHE INTEGER */ + 232, /* (75) replica ::= REPLICA INTEGER */ + 233, /* (76) quorum ::= QUORUM INTEGER */ + 234, /* (77) days ::= DAYS INTEGER */ + 235, /* (78) minrows ::= MINROWS INTEGER */ + 236, /* (79) maxrows ::= MAXROWS INTEGER */ + 237, /* (80) blocks ::= BLOCKS INTEGER */ + 238, /* (81) ctime ::= CTIME INTEGER */ + 239, /* (82) wal ::= WAL INTEGER */ + 240, /* (83) fsync ::= FSYNC INTEGER */ + 241, /* (84) comp ::= COMP INTEGER */ + 242, /* (85) prec ::= PRECISION STRING */ + 243, /* (86) update ::= UPDATE INTEGER */ + 244, /* (87) cachelast ::= CACHELAST INTEGER */ + 219, /* (88) db_optr ::= */ + 219, /* (89) db_optr ::= db_optr cache */ + 219, /* (90) db_optr ::= db_optr replica */ + 219, /* (91) db_optr ::= db_optr quorum */ + 219, /* (92) db_optr ::= db_optr days */ + 219, /* (93) db_optr ::= db_optr minrows */ + 219, /* (94) db_optr ::= db_optr maxrows */ + 219, /* (95) db_optr ::= db_optr blocks */ + 219, /* (96) db_optr ::= db_optr ctime */ + 219, /* (97) db_optr ::= db_optr wal */ + 219, /* (98) db_optr ::= db_optr fsync */ + 219, /* (99) db_optr ::= db_optr comp */ + 219, /* (100) db_optr ::= db_optr prec */ + 219, /* (101) db_optr ::= db_optr keep */ + 219, /* (102) db_optr ::= db_optr update */ + 219, /* (103) db_optr ::= db_optr cachelast */ + 216, /* (104) alter_db_optr ::= */ + 216, /* (105) alter_db_optr ::= alter_db_optr replica */ + 216, /* (106) alter_db_optr ::= alter_db_optr quorum */ + 216, /* (107) alter_db_optr ::= alter_db_optr keep */ + 216, /* (108) alter_db_optr ::= alter_db_optr blocks */ + 216, /* (109) alter_db_optr ::= alter_db_optr comp */ + 216, /* (110) alter_db_optr ::= alter_db_optr wal */ + 216, /* (111) alter_db_optr ::= alter_db_optr fsync */ + 216, /* (112) alter_db_optr ::= alter_db_optr update */ + 216, /* (113) alter_db_optr ::= alter_db_optr cachelast */ + 245, /* (114) typename ::= ids */ + 245, /* (115) typename ::= ids LP signed RP */ + 245, /* (116) typename ::= ids UNSIGNED */ + 246, /* (117) signed ::= INTEGER */ + 246, /* (118) signed ::= PLUS INTEGER */ + 246, /* (119) signed ::= MINUS INTEGER */ + 211, /* (120) cmd ::= CREATE TABLE create_table_args */ + 211, /* (121) cmd ::= CREATE TABLE create_stable_args */ + 211, /* (122) cmd ::= CREATE STABLE create_stable_args */ + 211, /* (123) cmd ::= CREATE TABLE create_table_list */ + 249, /* (124) create_table_list ::= create_from_stable */ + 249, /* (125) create_table_list ::= create_table_list create_from_stable */ + 247, /* (126) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + 248, /* (127) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + 250, /* (128) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + 250, /* (129) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ + 252, /* (130) tagNamelist ::= tagNamelist COMMA ids */ + 252, /* (131) tagNamelist ::= ids */ + 247, /* (132) create_table_args ::= ifnotexists ids cpxName AS select */ + 251, /* (133) columnlist ::= columnlist COMMA column */ + 251, /* (134) columnlist ::= column */ + 254, /* (135) column ::= ids typename */ + 230, /* (136) tagitemlist ::= tagitemlist COMMA tagitem */ + 230, /* (137) tagitemlist ::= tagitem */ + 255, /* (138) tagitem ::= INTEGER */ + 255, /* (139) tagitem ::= FLOAT */ + 255, /* (140) tagitem ::= STRING */ + 255, /* (141) tagitem ::= BOOL */ + 255, /* (142) tagitem ::= NULL */ + 255, /* (143) tagitem ::= MINUS INTEGER */ + 255, /* (144) tagitem ::= MINUS FLOAT */ + 255, /* (145) tagitem ::= PLUS INTEGER */ + 255, /* (146) tagitem ::= PLUS FLOAT */ + 253, /* (147) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + 267, /* (148) union ::= select */ + 267, /* (149) union ::= LP union RP */ + 267, /* (150) union ::= union UNION ALL select */ + 267, /* (151) union ::= union UNION ALL LP select RP */ + 211, /* (152) cmd ::= union */ + 253, /* (153) select ::= SELECT selcollist */ + 268, /* (154) sclp ::= selcollist COMMA */ + 268, /* (155) sclp ::= */ + 256, /* (156) selcollist ::= sclp distinct expr as */ + 256, /* (157) selcollist ::= sclp STAR */ + 271, /* (158) as ::= AS ids */ + 271, /* (159) as ::= ids */ + 271, /* (160) as ::= */ + 269, /* (161) distinct ::= DISTINCT */ + 269, /* (162) distinct ::= */ + 257, /* (163) from ::= FROM tablelist */ + 272, /* (164) tablelist ::= ids cpxName */ + 272, /* (165) tablelist ::= ids cpxName ids */ + 272, /* (166) tablelist ::= tablelist COMMA ids cpxName */ + 272, /* (167) tablelist ::= tablelist COMMA ids cpxName ids */ + 273, /* (168) tmvar ::= VARIABLE */ + 259, /* (169) interval_opt ::= INTERVAL LP tmvar RP */ + 259, /* (170) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + 259, /* (171) interval_opt ::= */ + 260, /* (172) fill_opt ::= */ + 260, /* (173) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + 260, /* (174) fill_opt ::= FILL LP ID RP */ + 261, /* (175) sliding_opt ::= SLIDING LP tmvar RP */ + 261, /* (176) sliding_opt ::= */ + 263, /* (177) orderby_opt ::= */ + 263, /* (178) orderby_opt ::= ORDER BY sortlist */ + 274, /* (179) sortlist ::= sortlist COMMA item sortorder */ + 274, /* (180) sortlist ::= item sortorder */ + 276, /* (181) item ::= ids cpxName */ + 277, /* (182) sortorder ::= ASC */ + 277, /* (183) sortorder ::= DESC */ + 277, /* (184) sortorder ::= */ + 262, /* (185) groupby_opt ::= */ + 262, /* (186) groupby_opt ::= GROUP BY grouplist */ + 278, /* (187) grouplist ::= grouplist COMMA item */ + 278, /* (188) grouplist ::= item */ + 264, /* (189) having_opt ::= */ + 264, /* (190) having_opt ::= HAVING expr */ + 266, /* (191) limit_opt ::= */ + 266, /* (192) limit_opt ::= LIMIT signed */ + 266, /* (193) limit_opt ::= LIMIT signed OFFSET signed */ + 266, /* (194) limit_opt ::= LIMIT signed COMMA signed */ + 265, /* (195) slimit_opt ::= */ + 265, /* (196) slimit_opt ::= SLIMIT signed */ + 265, /* (197) slimit_opt ::= SLIMIT signed SOFFSET signed */ + 265, /* (198) slimit_opt ::= SLIMIT signed COMMA signed */ + 258, /* (199) where_opt ::= */ + 258, /* (200) where_opt ::= WHERE expr */ + 270, /* (201) expr ::= LP expr RP */ + 270, /* (202) expr ::= ID */ + 270, /* (203) expr ::= ID DOT ID */ + 270, /* (204) expr ::= ID DOT STAR */ + 270, /* (205) expr ::= INTEGER */ + 270, /* (206) expr ::= MINUS INTEGER */ + 270, /* (207) expr ::= PLUS INTEGER */ + 270, /* (208) expr ::= FLOAT */ + 270, /* (209) expr ::= MINUS FLOAT */ + 270, /* (210) expr ::= PLUS FLOAT */ + 270, /* (211) expr ::= STRING */ + 270, /* (212) expr ::= NOW */ + 270, /* (213) expr ::= VARIABLE */ + 270, /* (214) expr ::= BOOL */ + 270, /* (215) expr ::= ID LP exprlist RP */ + 270, /* (216) expr ::= ID LP STAR RP */ + 270, /* (217) expr ::= expr IS NULL */ + 270, /* (218) expr ::= expr IS NOT NULL */ + 270, /* (219) expr ::= expr LT expr */ + 270, /* (220) expr ::= expr GT expr */ + 270, /* (221) expr ::= expr LE expr */ + 270, /* (222) expr ::= expr GE expr */ + 270, /* (223) expr ::= expr NE expr */ + 270, /* (224) expr ::= expr EQ expr */ + 270, /* (225) expr ::= expr BETWEEN expr AND expr */ + 270, /* (226) expr ::= expr AND expr */ + 270, /* (227) expr ::= expr OR expr */ + 270, /* (228) expr ::= expr PLUS expr */ + 270, /* (229) expr ::= expr MINUS expr */ + 270, /* (230) expr ::= expr STAR expr */ + 270, /* (231) expr ::= expr SLASH expr */ + 270, /* (232) expr ::= expr REM expr */ + 270, /* (233) expr ::= expr LIKE expr */ + 270, /* (234) expr ::= expr IN LP exprlist RP */ + 279, /* (235) exprlist ::= exprlist COMMA expritem */ + 279, /* (236) exprlist ::= expritem */ + 280, /* (237) expritem ::= expr */ + 280, /* (238) expritem ::= */ + 211, /* (239) cmd ::= RESET QUERY CACHE */ + 211, /* (240) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + 211, /* (241) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + 211, /* (242) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + 211, /* (243) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + 211, /* (244) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + 211, /* (245) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + 211, /* (246) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + 211, /* (247) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + 211, /* (248) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + 211, /* (249) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + 211, /* (250) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + 211, /* (251) cmd ::= KILL CONNECTION INTEGER */ + 211, /* (252) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + 211, /* (253) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { + -1, /* (0) program ::= cmd */ + -2, /* (1) cmd ::= SHOW DATABASES */ + -2, /* (2) cmd ::= SHOW MNODES */ + -2, /* (3) cmd ::= SHOW DNODES */ + -2, /* (4) cmd ::= SHOW ACCOUNTS */ + -2, /* (5) cmd ::= SHOW USERS */ + -2, /* (6) cmd ::= SHOW MODULES */ + -2, /* (7) cmd ::= SHOW QUERIES */ + -2, /* (8) cmd ::= SHOW CONNECTIONS */ + -2, /* (9) cmd ::= SHOW STREAMS */ + -2, /* (10) cmd ::= SHOW VARIABLES */ + -2, /* (11) cmd ::= SHOW SCORES */ + -2, /* (12) cmd ::= SHOW GRANTS */ + -2, /* (13) cmd ::= SHOW VNODES */ + -3, /* (14) cmd ::= SHOW VNODES IPTOKEN */ + 0, /* (15) dbPrefix ::= */ + -2, /* (16) dbPrefix ::= ids DOT */ + 0, /* (17) cpxName ::= */ + -2, /* (18) cpxName ::= DOT ids */ + -5, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ + -4, /* (20) cmd ::= SHOW CREATE DATABASE ids */ + -3, /* (21) cmd ::= SHOW dbPrefix TABLES */ + -5, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + -3, /* (23) cmd ::= SHOW dbPrefix STABLES */ + -5, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + -3, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ + -4, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ + -5, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ + -5, /* (28) cmd ::= DROP STABLE ifexists ids cpxName */ + -4, /* (29) cmd ::= DROP DATABASE ifexists ids */ + -3, /* (30) cmd ::= DROP DNODE ids */ + -3, /* (31) cmd ::= DROP USER ids */ + -3, /* (32) cmd ::= DROP ACCOUNT ids */ + -2, /* (33) cmd ::= USE ids */ + -3, /* (34) cmd ::= DESCRIBE ids cpxName */ + -5, /* (35) cmd ::= ALTER USER ids PASS ids */ + -5, /* (36) cmd ::= ALTER USER ids PRIVILEGE ids */ + -4, /* (37) cmd ::= ALTER DNODE ids ids */ + -5, /* (38) cmd ::= ALTER DNODE ids ids ids */ + -3, /* (39) cmd ::= ALTER LOCAL ids */ + -4, /* (40) cmd ::= ALTER LOCAL ids ids */ + -4, /* (41) cmd ::= ALTER DATABASE ids alter_db_optr */ + -4, /* (42) cmd ::= ALTER ACCOUNT ids acct_optr */ + -6, /* (43) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + -1, /* (44) ids ::= ID */ + -1, /* (45) ids ::= STRING */ + -2, /* (46) ifexists ::= IF EXISTS */ + 0, /* (47) ifexists ::= */ + -3, /* (48) ifnotexists ::= IF NOT EXISTS */ + 0, /* (49) ifnotexists ::= */ + -3, /* (50) cmd ::= CREATE DNODE ids */ + -6, /* (51) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + -5, /* (52) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + -5, /* (53) cmd ::= CREATE USER ids PASS ids */ + 0, /* (54) pps ::= */ + -2, /* (55) pps ::= PPS INTEGER */ + 0, /* (56) tseries ::= */ + -2, /* (57) tseries ::= TSERIES INTEGER */ + 0, /* (58) dbs ::= */ + -2, /* (59) dbs ::= DBS INTEGER */ + 0, /* (60) streams ::= */ + -2, /* (61) streams ::= STREAMS INTEGER */ + 0, /* (62) storage ::= */ + -2, /* (63) storage ::= STORAGE INTEGER */ + 0, /* (64) qtime ::= */ + -2, /* (65) qtime ::= QTIME INTEGER */ + 0, /* (66) users ::= */ + -2, /* (67) users ::= USERS INTEGER */ + 0, /* (68) conns ::= */ + -2, /* (69) conns ::= CONNS INTEGER */ + 0, /* (70) state ::= */ + -2, /* (71) state ::= STATE ids */ + -9, /* (72) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + -2, /* (73) keep ::= KEEP tagitemlist */ + -2, /* (74) cache ::= CACHE INTEGER */ + -2, /* (75) replica ::= REPLICA INTEGER */ + -2, /* (76) quorum ::= QUORUM INTEGER */ + -2, /* (77) days ::= DAYS INTEGER */ + -2, /* (78) minrows ::= MINROWS INTEGER */ + -2, /* (79) maxrows ::= MAXROWS INTEGER */ + -2, /* (80) blocks ::= BLOCKS INTEGER */ + -2, /* (81) ctime ::= CTIME INTEGER */ + -2, /* (82) wal ::= WAL INTEGER */ + -2, /* (83) fsync ::= FSYNC INTEGER */ + -2, /* (84) comp ::= COMP INTEGER */ + -2, /* (85) prec ::= PRECISION STRING */ + -2, /* (86) update ::= UPDATE INTEGER */ + -2, /* (87) cachelast ::= CACHELAST INTEGER */ + 0, /* (88) db_optr ::= */ + -2, /* (89) db_optr ::= db_optr cache */ + -2, /* (90) db_optr ::= db_optr replica */ + -2, /* (91) db_optr ::= db_optr quorum */ + -2, /* (92) db_optr ::= db_optr days */ + -2, /* (93) db_optr ::= db_optr minrows */ + -2, /* (94) db_optr ::= db_optr maxrows */ + -2, /* (95) db_optr ::= db_optr blocks */ + -2, /* (96) db_optr ::= db_optr ctime */ + -2, /* (97) db_optr ::= db_optr wal */ + -2, /* (98) db_optr ::= db_optr fsync */ + -2, /* (99) db_optr ::= db_optr comp */ + -2, /* (100) db_optr ::= db_optr prec */ + -2, /* (101) db_optr ::= db_optr keep */ + -2, /* (102) db_optr ::= db_optr update */ + -2, /* (103) db_optr ::= db_optr cachelast */ + 0, /* (104) alter_db_optr ::= */ + -2, /* (105) alter_db_optr ::= alter_db_optr replica */ + -2, /* (106) alter_db_optr ::= alter_db_optr quorum */ + -2, /* (107) alter_db_optr ::= alter_db_optr keep */ + -2, /* (108) alter_db_optr ::= alter_db_optr blocks */ + -2, /* (109) alter_db_optr ::= alter_db_optr comp */ + -2, /* (110) alter_db_optr ::= alter_db_optr wal */ + -2, /* (111) alter_db_optr ::= alter_db_optr fsync */ + -2, /* (112) alter_db_optr ::= alter_db_optr update */ + -2, /* (113) alter_db_optr ::= alter_db_optr cachelast */ + -1, /* (114) typename ::= ids */ + -4, /* (115) typename ::= ids LP signed RP */ + -2, /* (116) typename ::= ids UNSIGNED */ + -1, /* (117) signed ::= INTEGER */ + -2, /* (118) signed ::= PLUS INTEGER */ + -2, /* (119) signed ::= MINUS INTEGER */ + -3, /* (120) cmd ::= CREATE TABLE create_table_args */ + -3, /* (121) cmd ::= CREATE TABLE create_stable_args */ + -3, /* (122) cmd ::= CREATE STABLE create_stable_args */ + -3, /* (123) cmd ::= CREATE TABLE create_table_list */ + -1, /* (124) create_table_list ::= create_from_stable */ + -2, /* (125) create_table_list ::= create_table_list create_from_stable */ + -6, /* (126) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + -10, /* (127) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + -10, /* (128) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + -13, /* (129) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ + -3, /* (130) tagNamelist ::= tagNamelist COMMA ids */ + -1, /* (131) tagNamelist ::= ids */ + -5, /* (132) create_table_args ::= ifnotexists ids cpxName AS select */ + -3, /* (133) columnlist ::= columnlist COMMA column */ + -1, /* (134) columnlist ::= column */ + -2, /* (135) column ::= ids typename */ + -3, /* (136) tagitemlist ::= tagitemlist COMMA tagitem */ + -1, /* (137) tagitemlist ::= tagitem */ + -1, /* (138) tagitem ::= INTEGER */ + -1, /* (139) tagitem ::= FLOAT */ + -1, /* (140) tagitem ::= STRING */ + -1, /* (141) tagitem ::= BOOL */ + -1, /* (142) tagitem ::= NULL */ + -2, /* (143) tagitem ::= MINUS INTEGER */ + -2, /* (144) tagitem ::= MINUS FLOAT */ + -2, /* (145) tagitem ::= PLUS INTEGER */ + -2, /* (146) tagitem ::= PLUS FLOAT */ + -12, /* (147) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + -1, /* (148) union ::= select */ + -3, /* (149) union ::= LP union RP */ + -4, /* (150) union ::= union UNION ALL select */ + -6, /* (151) union ::= union UNION ALL LP select RP */ + -1, /* (152) cmd ::= union */ + -2, /* (153) select ::= SELECT selcollist */ + -2, /* (154) sclp ::= selcollist COMMA */ + 0, /* (155) sclp ::= */ + -4, /* (156) selcollist ::= sclp distinct expr as */ + -2, /* (157) selcollist ::= sclp STAR */ + -2, /* (158) as ::= AS ids */ + -1, /* (159) as ::= ids */ + 0, /* (160) as ::= */ + -1, /* (161) distinct ::= DISTINCT */ + 0, /* (162) distinct ::= */ + -2, /* (163) from ::= FROM tablelist */ + -2, /* (164) tablelist ::= ids cpxName */ + -3, /* (165) tablelist ::= ids cpxName ids */ + -4, /* (166) tablelist ::= tablelist COMMA ids cpxName */ + -5, /* (167) tablelist ::= tablelist COMMA ids cpxName ids */ + -1, /* (168) tmvar ::= VARIABLE */ + -4, /* (169) interval_opt ::= INTERVAL LP tmvar RP */ + -6, /* (170) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + 0, /* (171) interval_opt ::= */ + 0, /* (172) fill_opt ::= */ + -6, /* (173) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + -4, /* (174) fill_opt ::= FILL LP ID RP */ + -4, /* (175) sliding_opt ::= SLIDING LP tmvar RP */ + 0, /* (176) sliding_opt ::= */ + 0, /* (177) orderby_opt ::= */ + -3, /* (178) orderby_opt ::= ORDER BY sortlist */ + -4, /* (179) sortlist ::= sortlist COMMA item sortorder */ + -2, /* (180) sortlist ::= item sortorder */ + -2, /* (181) item ::= ids cpxName */ + -1, /* (182) sortorder ::= ASC */ + -1, /* (183) sortorder ::= DESC */ + 0, /* (184) sortorder ::= */ + 0, /* (185) groupby_opt ::= */ + -3, /* (186) groupby_opt ::= GROUP BY grouplist */ + -3, /* (187) grouplist ::= grouplist COMMA item */ + -1, /* (188) grouplist ::= item */ + 0, /* (189) having_opt ::= */ + -2, /* (190) having_opt ::= HAVING expr */ + 0, /* (191) limit_opt ::= */ + -2, /* (192) limit_opt ::= LIMIT signed */ + -4, /* (193) limit_opt ::= LIMIT signed OFFSET signed */ + -4, /* (194) limit_opt ::= LIMIT signed COMMA signed */ + 0, /* (195) slimit_opt ::= */ + -2, /* (196) slimit_opt ::= SLIMIT signed */ + -4, /* (197) slimit_opt ::= SLIMIT signed SOFFSET signed */ + -4, /* (198) slimit_opt ::= SLIMIT signed COMMA signed */ + 0, /* (199) where_opt ::= */ + -2, /* (200) where_opt ::= WHERE expr */ + -3, /* (201) expr ::= LP expr RP */ + -1, /* (202) expr ::= ID */ + -3, /* (203) expr ::= ID DOT ID */ + -3, /* (204) expr ::= ID DOT STAR */ + -1, /* (205) expr ::= INTEGER */ + -2, /* (206) expr ::= MINUS INTEGER */ + -2, /* (207) expr ::= PLUS INTEGER */ + -1, /* (208) expr ::= FLOAT */ + -2, /* (209) expr ::= MINUS FLOAT */ + -2, /* (210) expr ::= PLUS FLOAT */ + -1, /* (211) expr ::= STRING */ + -1, /* (212) expr ::= NOW */ + -1, /* (213) expr ::= VARIABLE */ + -1, /* (214) expr ::= BOOL */ + -4, /* (215) expr ::= ID LP exprlist RP */ + -4, /* (216) expr ::= ID LP STAR RP */ + -3, /* (217) expr ::= expr IS NULL */ + -4, /* (218) expr ::= expr IS NOT NULL */ + -3, /* (219) expr ::= expr LT expr */ + -3, /* (220) expr ::= expr GT expr */ + -3, /* (221) expr ::= expr LE expr */ + -3, /* (222) expr ::= expr GE expr */ + -3, /* (223) expr ::= expr NE expr */ + -3, /* (224) expr ::= expr EQ expr */ + -5, /* (225) expr ::= expr BETWEEN expr AND expr */ + -3, /* (226) expr ::= expr AND expr */ + -3, /* (227) expr ::= expr OR expr */ + -3, /* (228) expr ::= expr PLUS expr */ + -3, /* (229) expr ::= expr MINUS expr */ + -3, /* (230) expr ::= expr STAR expr */ + -3, /* (231) expr ::= expr SLASH expr */ + -3, /* (232) expr ::= expr REM expr */ + -3, /* (233) expr ::= expr LIKE expr */ + -5, /* (234) expr ::= expr IN LP exprlist RP */ + -3, /* (235) exprlist ::= exprlist COMMA expritem */ + -1, /* (236) exprlist ::= expritem */ + -1, /* (237) expritem ::= expr */ + 0, /* (238) expritem ::= */ + -3, /* (239) cmd ::= RESET QUERY CACHE */ + -7, /* (240) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (241) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + -7, /* (242) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + -7, /* (243) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + -8, /* (244) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + -9, /* (245) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + -7, /* (246) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (247) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + -7, /* (248) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + -7, /* (249) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + -8, /* (250) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + -3, /* (251) cmd ::= KILL CONNECTION INTEGER */ + -5, /* (252) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + -5, /* (253) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1945,30 +2304,34 @@ static void yy_accept(yyParser*); /* Forward Declaration */ ** only called from one place, optimizing compilers will in-line it, which ** means that the extra parameters have no performance impact. */ -static void yy_reduce( +static YYACTIONTYPE yy_reduce( yyParser *yypParser, /* The parser */ unsigned int yyruleno, /* Number of the rule by which to reduce */ int yyLookahead, /* Lookahead token, or YYNOCODE if none */ ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + ParseCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ - int yyact; /* The next action */ + YYACTIONTYPE yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - ParseARG_FETCH; + ParseARG_FETCH (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -1986,13 +2349,19 @@ static void yy_reduce( #if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ yyStackOverflow(yypParser); - return; + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; } #else if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); - return; + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; } yymsp = yypParser->yytos; } @@ -2011,6 +2380,9 @@ static void yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* program ::= cmd */ + case 120: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==120); + case 121: /* cmd ::= CREATE TABLE create_stable_args */ yytestcase(yyruleno==121); + case 122: /* cmd ::= CREATE STABLE create_stable_args */ yytestcase(yyruleno==122); {} break; case 1: /* cmd ::= SHOW DATABASES */ @@ -2097,730 +2469,807 @@ static void yy_reduce( case 24: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SStrToken token; - setDBName(&token, &yymsp[-3].minor.yy0); + setDbName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; case 25: /* cmd ::= SHOW dbPrefix VGROUPS */ { SStrToken token; - setDBName(&token, &yymsp[-1].minor.yy0); + setDbName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; case 26: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { SStrToken token; - setDBName(&token, &yymsp[-2].minor.yy0); + setDbName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= DROP TABLE ifexists ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDropDBTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0); + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1); +} + break; + case 28: /* cmd ::= DROP STABLE ifexists ids cpxName */ +{ + yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, TSDB_SUPER_TABLE); } break; - case 28: /* cmd ::= DROP DATABASE ifexists ids */ -{ setDropDBTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0); } + case 29: /* cmd ::= DROP DATABASE ifexists ids */ +{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, -1); } break; - case 29: /* cmd ::= DROP DNODE ids */ + case 30: /* cmd ::= DROP DNODE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } break; - case 30: /* cmd ::= DROP USER ids */ + case 31: /* cmd ::= DROP USER ids */ { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &yymsp[0].minor.yy0); } break; - case 31: /* cmd ::= DROP ACCOUNT ids */ + case 32: /* cmd ::= DROP ACCOUNT ids */ { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &yymsp[0].minor.yy0); } break; - case 32: /* cmd ::= USE ids */ + case 33: /* cmd ::= USE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_USE_DB, 1, &yymsp[0].minor.yy0);} break; - case 33: /* cmd ::= DESCRIBE ids cpxName */ + case 34: /* cmd ::= DESCRIBE ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; setDCLSQLElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &yymsp[-1].minor.yy0); } break; - case 34: /* cmd ::= ALTER USER ids PASS ids */ -{ setAlterUserSQL(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } + case 35: /* cmd ::= ALTER USER ids PASS ids */ +{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 35: /* cmd ::= ALTER USER ids PRIVILEGE ids */ -{ setAlterUserSQL(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} + case 36: /* cmd ::= ALTER USER ids PRIVILEGE ids */ +{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} break; - case 36: /* cmd ::= ALTER DNODE ids ids */ + case 37: /* cmd ::= ALTER DNODE ids ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 37: /* cmd ::= ALTER DNODE ids ids ids */ + case 38: /* cmd ::= ALTER DNODE ids ids ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 38: /* cmd ::= ALTER LOCAL ids */ + case 39: /* cmd ::= ALTER LOCAL ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &yymsp[0].minor.yy0); } break; - case 39: /* cmd ::= ALTER LOCAL ids ids */ + case 40: /* cmd ::= ALTER LOCAL ids ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 40: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} + case 41: /* cmd ::= ALTER DATABASE ids alter_db_optr */ +{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy262, &t);} break; - case 41: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} + case 42: /* cmd ::= ALTER ACCOUNT ids acct_optr */ +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy47);} break; - case 42: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} + case 43: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy47);} break; - case 43: /* ids ::= ID */ - case 44: /* ids ::= STRING */ yytestcase(yyruleno==44); + case 44: /* ids ::= ID */ + case 45: /* ids ::= STRING */ yytestcase(yyruleno==45); {yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 45: /* ifexists ::= IF EXISTS */ -{yymsp[-1].minor.yy0.n = 1;} + case 46: /* ifexists ::= IF EXISTS */ +{ yymsp[-1].minor.yy0.n = 1;} break; - case 46: /* ifexists ::= */ - case 48: /* ifnotexists ::= */ yytestcase(yyruleno==48); -{yymsp[1].minor.yy0.n = 0;} + case 47: /* ifexists ::= */ + case 49: /* ifnotexists ::= */ yytestcase(yyruleno==49); + case 162: /* distinct ::= */ yytestcase(yyruleno==162); +{ yymsp[1].minor.yy0.n = 0;} break; - case 47: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy0.n = 1;} + case 48: /* ifnotexists ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy0.n = 1;} break; - case 49: /* cmd ::= CREATE DNODE ids */ + case 50: /* cmd ::= CREATE DNODE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; - case 50: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} - break; - case 51: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);} - break; - case 52: /* cmd ::= CREATE USER ids PASS ids */ -{ setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} - break; - case 53: /* pps ::= */ - case 55: /* tseries ::= */ yytestcase(yyruleno==55); - case 57: /* dbs ::= */ yytestcase(yyruleno==57); - case 59: /* streams ::= */ yytestcase(yyruleno==59); - case 61: /* storage ::= */ yytestcase(yyruleno==61); - case 63: /* qtime ::= */ yytestcase(yyruleno==63); - case 65: /* users ::= */ yytestcase(yyruleno==65); - case 67: /* conns ::= */ yytestcase(yyruleno==67); - case 69: /* state ::= */ yytestcase(yyruleno==69); -{yymsp[1].minor.yy0.n = 0; } - break; - case 54: /* pps ::= PPS INTEGER */ - case 56: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==56); - case 58: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==58); - case 60: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==60); - case 62: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==62); - case 64: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==64); - case 66: /* users ::= USERS INTEGER */ yytestcase(yyruleno==66); - case 68: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==68); - case 70: /* state ::= STATE ids */ yytestcase(yyruleno==70); -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } - break; - case 71: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + case 51: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ +{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy47);} + break; + case 52: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ +{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy262, &yymsp[-2].minor.yy0);} + break; + case 53: /* cmd ::= CREATE USER ids PASS ids */ +{ setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 54: /* pps ::= */ + case 56: /* tseries ::= */ yytestcase(yyruleno==56); + case 58: /* dbs ::= */ yytestcase(yyruleno==58); + case 60: /* streams ::= */ yytestcase(yyruleno==60); + case 62: /* storage ::= */ yytestcase(yyruleno==62); + case 64: /* qtime ::= */ yytestcase(yyruleno==64); + case 66: /* users ::= */ yytestcase(yyruleno==66); + case 68: /* conns ::= */ yytestcase(yyruleno==68); + case 70: /* state ::= */ yytestcase(yyruleno==70); +{ yymsp[1].minor.yy0.n = 0; } + break; + case 55: /* pps ::= PPS INTEGER */ + case 57: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==57); + case 59: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==59); + case 61: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==61); + case 63: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==63); + case 65: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==65); + case 67: /* users ::= USERS INTEGER */ yytestcase(yyruleno==67); + case 69: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==69); + case 71: /* state ::= STATE ids */ yytestcase(yyruleno==71); +{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } + break; + case 72: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy149.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy149.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy149.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy149.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy149.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy149.stat = yymsp[0].minor.yy0; -} - yymsp[-8].minor.yy149 = yylhsminor.yy149; - break; - case 72: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy165 = yymsp[0].minor.yy165; } - break; - case 73: /* cache ::= CACHE INTEGER */ - case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74); - case 75: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==75); - case 76: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==76); - case 77: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==77); - case 78: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==78); - case 79: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==79); - case 80: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==80); - case 81: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==81); - case 82: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==82); - case 83: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==83); - case 84: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==84); - case 85: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==85); + yylhsminor.yy47.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy47.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy47.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy47.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy47.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy47.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy47.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy47.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy47.stat = yymsp[0].minor.yy0; +} + yymsp[-8].minor.yy47 = yylhsminor.yy47; + break; + case 73: /* keep ::= KEEP tagitemlist */ +{ yymsp[-1].minor.yy247 = yymsp[0].minor.yy247; } + break; + case 74: /* cache ::= CACHE INTEGER */ + case 75: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==75); + case 76: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==76); + case 77: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==77); + case 78: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==78); + case 79: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==79); + case 80: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==80); + case 81: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==81); + case 82: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==82); + case 83: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==83); + case 84: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==84); + case 85: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==85); + case 86: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==86); + case 87: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==87); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 86: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy268);} - break; - case 87: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 88: /* db_optr ::= db_optr replica */ - case 102: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==102); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 89: /* db_optr ::= db_optr quorum */ - case 103: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==103); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 90: /* db_optr ::= db_optr days */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 91: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 92: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 93: /* db_optr ::= db_optr blocks */ - case 105: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==105); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 94: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 95: /* db_optr ::= db_optr wal */ - case 107: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==107); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 96: /* db_optr ::= db_optr fsync */ - case 108: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==108); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 97: /* db_optr ::= db_optr comp */ - case 106: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==106); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 98: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 99: /* db_optr ::= db_optr keep */ - case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy165; } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 100: /* db_optr ::= db_optr update */ - case 109: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==109); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 101: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy268);} - break; - case 110: /* typename ::= ids */ + case 88: /* db_optr ::= */ +{setDefaultCreateDbOption(&yymsp[1].minor.yy262);} + break; + case 89: /* db_optr ::= db_optr cache */ +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 90: /* db_optr ::= db_optr replica */ + case 105: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==105); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 91: /* db_optr ::= db_optr quorum */ + case 106: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==106); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 92: /* db_optr ::= db_optr days */ +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 93: /* db_optr ::= db_optr minrows */ +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 94: /* db_optr ::= db_optr maxrows */ +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 95: /* db_optr ::= db_optr blocks */ + case 108: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==108); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 96: /* db_optr ::= db_optr ctime */ +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 97: /* db_optr ::= db_optr wal */ + case 110: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==110); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 98: /* db_optr ::= db_optr fsync */ + case 111: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==111); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 99: /* db_optr ::= db_optr comp */ + case 109: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==109); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 100: /* db_optr ::= db_optr prec */ +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 101: /* db_optr ::= db_optr keep */ + case 107: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==107); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.keep = yymsp[0].minor.yy247; } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 102: /* db_optr ::= db_optr update */ + case 112: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==112); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 103: /* db_optr ::= db_optr cachelast */ + case 113: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==113); +{ yylhsminor.yy262 = yymsp[-1].minor.yy262; yylhsminor.yy262.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy262 = yylhsminor.yy262; + break; + case 104: /* alter_db_optr ::= */ +{ setDefaultCreateDbOption(&yymsp[1].minor.yy262);} + break; + case 114: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSQLSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0); + tSqlSetColumnType (&yylhsminor.yy179, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy223 = yylhsminor.yy223; + yymsp[0].minor.yy179 = yylhsminor.yy179; break; - case 111: /* typename ::= ids LP signed RP */ + case 115: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy207 <= 0) { - yymsp[-3].minor.yy0.type = 0; - tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); - } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207; // negative value of name length - tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); - } + if (yymsp[-1].minor.yy403 <= 0) { + yymsp[-3].minor.yy0.type = 0; + tSqlSetColumnType(&yylhsminor.yy179, &yymsp[-3].minor.yy0); + } else { + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy403; // negative value of name length + tSqlSetColumnType(&yylhsminor.yy179, &yymsp[-3].minor.yy0); + } +} + yymsp[-3].minor.yy179 = yylhsminor.yy179; + break; + case 116: /* typename ::= ids UNSIGNED */ +{ + yymsp[-1].minor.yy0.type = 0; + yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); + tSqlSetColumnType (&yylhsminor.yy179, &yymsp[-1].minor.yy0); } - yymsp[-3].minor.yy223 = yylhsminor.yy223; + yymsp[-1].minor.yy179 = yylhsminor.yy179; break; - case 112: /* signed ::= INTEGER */ -{ yylhsminor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy207 = yylhsminor.yy207; + case 117: /* signed ::= INTEGER */ +{ yylhsminor.yy403 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy403 = yylhsminor.yy403; break; - case 113: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 118: /* signed ::= PLUS INTEGER */ +{ yymsp[-1].minor.yy403 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 114: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} + case 119: /* signed ::= MINUS INTEGER */ +{ yymsp[-1].minor.yy403 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 115: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + case 123: /* cmd ::= CREATE TABLE create_table_list */ +{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy358;} + break; + case 124: /* create_table_list ::= create_from_stable */ { - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); + SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); + pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); + + taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy42); + pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; + yylhsminor.yy358 = pCreateTable; } + yymsp[0].minor.yy358 = yylhsminor.yy358; break; - case 116: /* create_table_args ::= LP columnlist RP */ + case 125: /* create_table_list ::= create_table_list create_from_stable */ { - yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy165, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + taosArrayPush(yymsp[-1].minor.yy358->childTableInfo, &yymsp[0].minor.yy42); + yylhsminor.yy358 = yymsp[-1].minor.yy358; } + yymsp[-1].minor.yy358 = yylhsminor.yy358; break; - case 117: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + case 126: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { - yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy165, yymsp[-1].minor.yy165, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy358 = tSetCreateSqlElems(yymsp[-1].minor.yy247, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, yylhsminor.yy358, NULL, TSDB_SQL_CREATE_TABLE); + + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); } + yymsp[-5].minor.yy358 = yylhsminor.yy358; break; - case 118: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + case 127: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy165, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy358 = tSetCreateSqlElems(yymsp[-5].minor.yy247, yymsp[-1].minor.yy247, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, yylhsminor.yy358, NULL, TSDB_SQL_CREATE_TABLE); + + yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; + setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } + yymsp[-9].minor.yy358 = yylhsminor.yy358; break; - case 119: /* create_table_args ::= AS select */ + case 128: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ { - yymsp[-1].minor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, yymsp[-1].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; + yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; + yylhsminor.yy42 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy247, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } + yymsp[-9].minor.yy42 = yylhsminor.yy42; break; - case 120: /* columnlist ::= columnlist COMMA column */ -{taosArrayPush(yymsp[-2].minor.yy165, &yymsp[0].minor.yy223); yylhsminor.yy165 = yymsp[-2].minor.yy165; } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + case 129: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ +{ + yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; + yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; + yylhsminor.yy42 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy247, yymsp[-1].minor.yy247, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); +} + yymsp[-12].minor.yy42 = yylhsminor.yy42; + break; + case 130: /* tagNamelist ::= tagNamelist COMMA ids */ +{taosArrayPush(yymsp[-2].minor.yy247, &yymsp[0].minor.yy0); yylhsminor.yy247 = yymsp[-2].minor.yy247; } + yymsp[-2].minor.yy247 = yylhsminor.yy247; break; - case 121: /* columnlist ::= column */ -{yylhsminor.yy165 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy165, &yymsp[0].minor.yy223);} - yymsp[0].minor.yy165 = yylhsminor.yy165; + case 131: /* tagNamelist ::= ids */ +{yylhsminor.yy247 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy247, &yymsp[0].minor.yy0);} + yymsp[0].minor.yy247 = yylhsminor.yy247; break; - case 122: /* column ::= ids typename */ + case 132: /* create_table_args ::= ifnotexists ids cpxName AS select */ { - tSQLSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223); + yylhsminor.yy358 = tSetCreateSqlElems(NULL, NULL, yymsp[0].minor.yy114, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, yylhsminor.yy358, NULL, TSDB_SQL_CREATE_TABLE); + + yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; + setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); } - yymsp[-1].minor.yy223 = yylhsminor.yy223; + yymsp[-4].minor.yy358 = yylhsminor.yy358; break; - case 123: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1); } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + case 133: /* columnlist ::= columnlist COMMA column */ +{taosArrayPush(yymsp[-2].minor.yy247, &yymsp[0].minor.yy179); yylhsminor.yy247 = yymsp[-2].minor.yy247; } + yymsp[-2].minor.yy247 = yylhsminor.yy247; break; - case 124: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } - yymsp[0].minor.yy165 = yylhsminor.yy165; + case 134: /* columnlist ::= column */ +{yylhsminor.yy247 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy247, &yymsp[0].minor.yy179);} + yymsp[0].minor.yy247 = yylhsminor.yy247; + break; + case 135: /* column ::= ids typename */ +{ + tSqlSetColumnInfo(&yylhsminor.yy179, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy179); +} + yymsp[-1].minor.yy179 = yylhsminor.yy179; break; - case 125: /* tagitem ::= INTEGER */ - case 126: /* tagitem ::= FLOAT */ yytestcase(yyruleno==126); - case 127: /* tagitem ::= STRING */ yytestcase(yyruleno==127); - case 128: /* tagitem ::= BOOL */ yytestcase(yyruleno==128); -{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy134 = yylhsminor.yy134; + case 136: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yylhsminor.yy247 = tVariantListAppend(yymsp[-2].minor.yy247, &yymsp[0].minor.yy378, -1); } + yymsp[-2].minor.yy247 = yylhsminor.yy247; break; - case 129: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy134 = yylhsminor.yy134; + case 137: /* tagitemlist ::= tagitem */ +{ yylhsminor.yy247 = tVariantListAppend(NULL, &yymsp[0].minor.yy378, -1); } + yymsp[0].minor.yy247 = yylhsminor.yy247; break; - case 130: /* tagitem ::= MINUS INTEGER */ - case 131: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==131); - case 132: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==132); - case 133: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==133); + case 138: /* tagitem ::= INTEGER */ + case 139: /* tagitem ::= FLOAT */ yytestcase(yyruleno==139); + case 140: /* tagitem ::= STRING */ yytestcase(yyruleno==140); + case 141: /* tagitem ::= BOOL */ yytestcase(yyruleno==141); +{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy378, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy378 = yylhsminor.yy378; + break; + case 142: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy378, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy378 = yylhsminor.yy378; + break; + case 143: /* tagitem ::= MINUS INTEGER */ + case 144: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==144); + case 145: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==145); + case 146: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==146); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy378, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy134 = yylhsminor.yy134; + yymsp[-1].minor.yy378 = yylhsminor.yy378; break; - case 134: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 147: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy165, yymsp[-8].minor.yy64, yymsp[-4].minor.yy165, yymsp[-3].minor.yy165, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy165, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216); + yylhsminor.yy114 = tSetQuerySqlElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy522, yymsp[-9].minor.yy247, yymsp[-8].minor.yy326, yymsp[-4].minor.yy247, yymsp[-3].minor.yy247, &yymsp[-7].minor.yy430, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy247, &yymsp[0].minor.yy204, &yymsp[-1].minor.yy204); } - yymsp[-11].minor.yy414 = yylhsminor.yy414; + yymsp[-11].minor.yy114 = yylhsminor.yy114; break; - case 135: /* union ::= select */ -{ yylhsminor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); } - yymsp[0].minor.yy231 = yylhsminor.yy231; + case 148: /* union ::= select */ +{ yylhsminor.yy219 = setSubclause(NULL, yymsp[0].minor.yy114); } + yymsp[0].minor.yy219 = yylhsminor.yy219; break; - case 136: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy231 = yymsp[-1].minor.yy231; } + case 149: /* union ::= LP union RP */ +{ yymsp[-2].minor.yy219 = yymsp[-1].minor.yy219; } break; - case 137: /* union ::= union UNION ALL select */ -{ yylhsminor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); } - yymsp[-3].minor.yy231 = yylhsminor.yy231; + case 150: /* union ::= union UNION ALL select */ +{ yylhsminor.yy219 = appendSelectClause(yymsp[-3].minor.yy219, yymsp[0].minor.yy114); } + yymsp[-3].minor.yy219 = yylhsminor.yy219; break; - case 138: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); } - yymsp[-5].minor.yy231 = yylhsminor.yy231; + case 151: /* union ::= union UNION ALL LP select RP */ +{ yylhsminor.yy219 = appendSelectClause(yymsp[-5].minor.yy219, yymsp[-1].minor.yy114); } + yymsp[-5].minor.yy219 = yylhsminor.yy219; break; - case 139: /* cmd ::= union */ -{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); } + case 152: /* cmd ::= union */ +{ setSqlInfo(pInfo, yymsp[0].minor.yy219, NULL, TSDB_SQL_SELECT); } break; - case 140: /* select ::= SELECT selcollist */ + case 153: /* select ::= SELECT selcollist */ { - yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy114 = tSetQuerySqlElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy522, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy414 = yylhsminor.yy414; + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; - case 141: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy290 = yymsp[-1].minor.yy290;} - yymsp[-1].minor.yy290 = yylhsminor.yy290; + case 154: /* sclp ::= selcollist COMMA */ +{yylhsminor.yy522 = yymsp[-1].minor.yy522;} + yymsp[-1].minor.yy522 = yylhsminor.yy522; break; - case 142: /* sclp ::= */ -{yymsp[1].minor.yy290 = 0;} + case 155: /* sclp ::= */ +{yymsp[1].minor.yy522 = 0;} break; - case 143: /* selcollist ::= sclp expr as */ + case 156: /* selcollist ::= sclp distinct expr as */ { - yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy522 = tSqlExprListAppend(yymsp[-3].minor.yy522, yymsp[-1].minor.yy326, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy290 = yylhsminor.yy290; + yymsp[-3].minor.yy522 = yylhsminor.yy522; break; - case 144: /* selcollist ::= sclp STAR */ + case 157: /* selcollist ::= sclp STAR */ { - tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0); + tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); + yylhsminor.yy522 = tSqlExprListAppend(yymsp[-1].minor.yy522, pNode, 0, 0); } - yymsp[-1].minor.yy290 = yylhsminor.yy290; + yymsp[-1].minor.yy522 = yylhsminor.yy522; break; - case 145: /* as ::= AS ids */ + case 158: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 146: /* as ::= ids */ + case 159: /* as ::= ids */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 147: /* as ::= */ + case 160: /* as ::= */ { yymsp[1].minor.yy0.n = 0; } break; - case 148: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy165 = yymsp[0].minor.yy165;} + case 161: /* distinct ::= DISTINCT */ +{ yylhsminor.yy0 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 149: /* tablelist ::= ids cpxName */ + case 163: /* from ::= FROM tablelist */ +{yymsp[-1].minor.yy247 = yymsp[0].minor.yy247;} + break; + case 164: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1); // table alias name + yylhsminor.yy247 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy247 = tVariantListAppendToken(yylhsminor.yy247, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy165 = yylhsminor.yy165; + yymsp[-1].minor.yy247 = yylhsminor.yy247; break; - case 150: /* tablelist ::= ids cpxName ids */ + case 165: /* tablelist ::= ids cpxName ids */ { - toTSDBType(yymsp[-2].minor.yy0.type); - toTSDBType(yymsp[0].minor.yy0.type); - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1); + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy247 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy247 = tVariantListAppendToken(yylhsminor.yy247, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + yymsp[-2].minor.yy247 = yylhsminor.yy247; break; - case 151: /* tablelist ::= tablelist COMMA ids cpxName */ + case 166: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy247 = tVariantListAppendToken(yymsp[-3].minor.yy247, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy247 = tVariantListAppendToken(yylhsminor.yy247, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy165 = yylhsminor.yy165; + yymsp[-3].minor.yy247 = yylhsminor.yy247; break; - case 152: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 167: /* tablelist ::= tablelist COMMA ids cpxName ids */ { - toTSDBType(yymsp[-2].minor.yy0.type); - toTSDBType(yymsp[0].minor.yy0.type); - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(yymsp[-4].minor.yy165, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1); + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy247 = tVariantListAppendToken(yymsp[-4].minor.yy247, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy247 = tVariantListAppendToken(yylhsminor.yy247, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy165 = yylhsminor.yy165; + yymsp[-4].minor.yy247 = yylhsminor.yy247; break; - case 153: /* tmvar ::= VARIABLE */ + case 168: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 154: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy532.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy532.offset.n = 0; yymsp[-3].minor.yy532.offset.z = NULL; yymsp[-3].minor.yy532.offset.type = 0;} + case 169: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yymsp[-3].minor.yy430.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy430.offset.n = 0; yymsp[-3].minor.yy430.offset.z = NULL; yymsp[-3].minor.yy430.offset.type = 0;} break; - case 155: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy532.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy532.offset = yymsp[-1].minor.yy0;} + case 170: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yymsp[-5].minor.yy430.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy430.offset = yymsp[-1].minor.yy0;} break; - case 156: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy532, 0, sizeof(yymsp[1].minor.yy532));} + case 171: /* interval_opt ::= */ +{memset(&yymsp[1].minor.yy430, 0, sizeof(yymsp[1].minor.yy430));} break; - case 157: /* fill_opt ::= */ -{yymsp[1].minor.yy165 = 0; } + case 172: /* fill_opt ::= */ +{yymsp[1].minor.yy247 = 0; } break; - case 158: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 173: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy165, &A, -1, 0); - yymsp[-5].minor.yy165 = yymsp[-1].minor.yy165; + tVariantListInsert(yymsp[-1].minor.yy247, &A, -1, 0); + yymsp[-5].minor.yy247 = yymsp[-1].minor.yy247; } break; - case 159: /* fill_opt ::= FILL LP ID RP */ + case 174: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy247 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 160: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 175: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 161: /* sliding_opt ::= */ + case 176: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 162: /* orderby_opt ::= */ - case 170: /* groupby_opt ::= */ yytestcase(yyruleno==170); -{yymsp[1].minor.yy165 = 0;} + case 177: /* orderby_opt ::= */ +{yymsp[1].minor.yy247 = 0;} break; - case 163: /* orderby_opt ::= ORDER BY sortlist */ - case 171: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==171); -{yymsp[-2].minor.yy165 = yymsp[0].minor.yy165;} + case 178: /* orderby_opt ::= ORDER BY sortlist */ +{yymsp[-2].minor.yy247 = yymsp[0].minor.yy247;} break; - case 164: /* sortlist ::= sortlist COMMA item sortorder */ + case 179: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy165 = tVariantListAppend(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); + yylhsminor.yy247 = tVariantListAppend(yymsp[-3].minor.yy247, &yymsp[-1].minor.yy378, yymsp[0].minor.yy222); } - yymsp[-3].minor.yy165 = yylhsminor.yy165; + yymsp[-3].minor.yy247 = yylhsminor.yy247; break; - case 165: /* sortlist ::= item sortorder */ + case 180: /* sortlist ::= item sortorder */ { - yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); + yylhsminor.yy247 = tVariantListAppend(NULL, &yymsp[-1].minor.yy378, yymsp[0].minor.yy222); } - yymsp[-1].minor.yy165 = yylhsminor.yy165; + yymsp[-1].minor.yy247 = yylhsminor.yy247; break; - case 166: /* item ::= ids cpxName */ + case 181: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy378, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy134 = yylhsminor.yy134; + yymsp[-1].minor.yy378 = yylhsminor.yy378; + break; + case 182: /* sortorder ::= ASC */ +{ yymsp[0].minor.yy222 = TSDB_ORDER_ASC; } break; - case 167: /* sortorder ::= ASC */ -{yymsp[0].minor.yy46 = TSDB_ORDER_ASC; } + case 183: /* sortorder ::= DESC */ +{ yymsp[0].minor.yy222 = TSDB_ORDER_DESC;} break; - case 168: /* sortorder ::= DESC */ -{yymsp[0].minor.yy46 = TSDB_ORDER_DESC;} + case 184: /* sortorder ::= */ +{ yymsp[1].minor.yy222 = TSDB_ORDER_ASC; } break; - case 169: /* sortorder ::= */ -{yymsp[1].minor.yy46 = TSDB_ORDER_ASC;} + case 185: /* groupby_opt ::= */ +{ yymsp[1].minor.yy247 = 0;} break; - case 172: /* grouplist ::= grouplist COMMA item */ + case 186: /* groupby_opt ::= GROUP BY grouplist */ +{ yymsp[-2].minor.yy247 = yymsp[0].minor.yy247;} + break; + case 187: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1); + yylhsminor.yy247 = tVariantListAppend(yymsp[-2].minor.yy247, &yymsp[0].minor.yy378, -1); } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + yymsp[-2].minor.yy247 = yylhsminor.yy247; break; - case 173: /* grouplist ::= item */ + case 188: /* grouplist ::= item */ { - yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); + yylhsminor.yy247 = tVariantListAppend(NULL, &yymsp[0].minor.yy378, -1); } - yymsp[0].minor.yy165 = yylhsminor.yy165; + yymsp[0].minor.yy247 = yylhsminor.yy247; + break; + case 189: /* having_opt ::= */ + case 199: /* where_opt ::= */ yytestcase(yyruleno==199); + case 238: /* expritem ::= */ yytestcase(yyruleno==238); +{yymsp[1].minor.yy326 = 0;} break; - case 174: /* having_opt ::= */ - case 184: /* where_opt ::= */ yytestcase(yyruleno==184); - case 222: /* expritem ::= */ yytestcase(yyruleno==222); -{yymsp[1].minor.yy64 = 0;} + case 190: /* having_opt ::= HAVING expr */ + case 200: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==200); +{yymsp[-1].minor.yy326 = yymsp[0].minor.yy326;} break; - case 175: /* having_opt ::= HAVING expr */ - case 185: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==185); -{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;} + case 191: /* limit_opt ::= */ + case 195: /* slimit_opt ::= */ yytestcase(yyruleno==195); +{yymsp[1].minor.yy204.limit = -1; yymsp[1].minor.yy204.offset = 0;} break; - case 176: /* limit_opt ::= */ - case 180: /* slimit_opt ::= */ yytestcase(yyruleno==180); -{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;} + case 192: /* limit_opt ::= LIMIT signed */ + case 196: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==196); +{yymsp[-1].minor.yy204.limit = yymsp[0].minor.yy403; yymsp[-1].minor.yy204.offset = 0;} break; - case 177: /* limit_opt ::= LIMIT signed */ - case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181); -{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} + case 193: /* limit_opt ::= LIMIT signed OFFSET signed */ +{ yymsp[-3].minor.yy204.limit = yymsp[-2].minor.yy403; yymsp[-3].minor.yy204.offset = yymsp[0].minor.yy403;} break; - case 178: /* limit_opt ::= LIMIT signed OFFSET signed */ -{ yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} + case 194: /* limit_opt ::= LIMIT signed COMMA signed */ +{ yymsp[-3].minor.yy204.limit = yymsp[0].minor.yy403; yymsp[-3].minor.yy204.offset = yymsp[-2].minor.yy403;} break; - case 179: /* limit_opt ::= LIMIT signed COMMA signed */ -{ yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} + case 197: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ +{yymsp[-3].minor.yy204.limit = yymsp[-2].minor.yy403; yymsp[-3].minor.yy204.offset = yymsp[0].minor.yy403;} break; - case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} + case 198: /* slimit_opt ::= SLIMIT signed COMMA signed */ +{yymsp[-3].minor.yy204.limit = yymsp[0].minor.yy403; yymsp[-3].minor.yy204.offset = yymsp[-2].minor.yy403;} break; - case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */ -{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} + case 201: /* expr ::= LP expr RP */ +{yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy326->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 186: /* expr ::= LP expr RP */ -{yylhsminor.yy64 = yymsp[-1].minor.yy64; yylhsminor.yy64->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy64->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 202: /* expr ::= ID */ +{ yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 187: /* expr ::= ID */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 203: /* expr ::= ID DOT ID */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 188: /* expr ::= ID DOT ID */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 204: /* expr ::= ID DOT STAR */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 189: /* expr ::= ID DOT STAR */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 205: /* expr ::= INTEGER */ +{ yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 190: /* expr ::= INTEGER */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 206: /* expr ::= MINUS INTEGER */ + case 207: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==207); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy326 = yylhsminor.yy326; break; - case 191: /* expr ::= MINUS INTEGER */ - case 192: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==192); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy64 = yylhsminor.yy64; + case 208: /* expr ::= FLOAT */ +{ yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 193: /* expr ::= FLOAT */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 209: /* expr ::= MINUS FLOAT */ + case 210: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==210); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy326 = yylhsminor.yy326; break; - case 194: /* expr ::= MINUS FLOAT */ - case 195: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==195); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy64 = yylhsminor.yy64; + case 211: /* expr ::= STRING */ +{ yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 196: /* expr ::= STRING */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 212: /* expr ::= NOW */ +{ yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 197: /* expr ::= NOW */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 213: /* expr ::= VARIABLE */ +{ yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 198: /* expr ::= VARIABLE */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 214: /* expr ::= BOOL */ +{ yylhsminor.yy326 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 199: /* expr ::= BOOL */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 215: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy326 = tSqlExprCreateFunction(yymsp[-1].minor.yy522, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy326 = yylhsminor.yy326; break; - case 200: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy64 = yylhsminor.yy64; + case 216: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy326 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy326 = yylhsminor.yy326; break; - case 201: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy64 = yylhsminor.yy64; + case 217: /* expr ::= expr IS NULL */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, NULL, TK_ISNULL);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 202: /* expr ::= expr IS NULL */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, NULL, TK_ISNULL);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 218: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-3].minor.yy326, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy326 = yylhsminor.yy326; break; - case 203: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-3].minor.yy64, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy64 = yylhsminor.yy64; + case 219: /* expr ::= expr LT expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_LT);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 204: /* expr ::= expr LT expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 220: /* expr ::= expr GT expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_GT);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 205: /* expr ::= expr GT expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 221: /* expr ::= expr LE expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_LE);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 206: /* expr ::= expr LE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 222: /* expr ::= expr GE expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_GE);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 207: /* expr ::= expr GE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 223: /* expr ::= expr NE expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_NE);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 208: /* expr ::= expr NE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 224: /* expr ::= expr EQ expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_EQ);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 209: /* expr ::= expr EQ expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 225: /* expr ::= expr BETWEEN expr AND expr */ +{ tSQLExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy326); yylhsminor.yy326 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy326, yymsp[-2].minor.yy326, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy326, TK_LE), TK_AND);} + yymsp[-4].minor.yy326 = yylhsminor.yy326; break; - case 210: /* expr ::= expr AND expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 226: /* expr ::= expr AND expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_AND);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 211: /* expr ::= expr OR expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 227: /* expr ::= expr OR expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_OR); } + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 212: /* expr ::= expr PLUS expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 228: /* expr ::= expr PLUS expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_PLUS); } + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 213: /* expr ::= expr MINUS expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 229: /* expr ::= expr MINUS expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_MINUS); } + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 214: /* expr ::= expr STAR expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 230: /* expr ::= expr STAR expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_STAR); } + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 215: /* expr ::= expr SLASH expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 231: /* expr ::= expr SLASH expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_DIVIDE);} + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 216: /* expr ::= expr REM expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 232: /* expr ::= expr REM expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_REM); } + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 217: /* expr ::= expr LIKE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 233: /* expr ::= expr LIKE expr */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-2].minor.yy326, yymsp[0].minor.yy326, TK_LIKE); } + yymsp[-2].minor.yy326 = yylhsminor.yy326; break; - case 218: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); } - yymsp[-4].minor.yy64 = yylhsminor.yy64; + case 234: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy326 = tSqlExprCreate(yymsp[-4].minor.yy326, (tSQLExpr*)yymsp[-1].minor.yy522, TK_IN); } + yymsp[-4].minor.yy326 = yylhsminor.yy326; break; - case 219: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);} - yymsp[-2].minor.yy290 = yylhsminor.yy290; + case 235: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy522 = tSqlExprListAppend(yymsp[-2].minor.yy522,yymsp[0].minor.yy326,0, 0);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 220: /* exprlist ::= expritem */ -{yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);} - yymsp[0].minor.yy290 = yylhsminor.yy290; + case 236: /* exprlist ::= expritem */ +{yylhsminor.yy522 = tSqlExprListAppend(0,yymsp[0].minor.yy326,0, 0);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 221: /* expritem ::= expr */ -{yylhsminor.yy64 = yymsp[0].minor.yy64;} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 237: /* expritem ::= expr */ +{yylhsminor.yy326 = yymsp[0].minor.yy326;} + yymsp[0].minor.yy326 = yylhsminor.yy326; break; - case 223: /* cmd ::= RESET QUERY CACHE */ + case 239: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 224: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 240: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy247, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 225: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 241: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; toTSDBType(yymsp[0].minor.yy0.type); SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 226: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 242: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy247, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 227: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 243: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; toTSDBType(yymsp[0].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 228: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 244: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2830,38 +3279,88 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 229: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 245: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy378, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 246: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy247, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 230: /* cmd ::= KILL CONNECTION INTEGER */ -{setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} + case 247: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + + toTSDBType(yymsp[0].minor.yy0.type); + SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); + + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} break; - case 231: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} + case 248: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy247, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} break; - case 232: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} + case 249: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + + toTSDBType(yymsp[0].minor.yy0.type); + SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); + + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 250: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ +{ + yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; + + toTSDBType(yymsp[-1].minor.yy0.type); + SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + + toTSDBType(yymsp[0].minor.yy0.type); + A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); + + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 251: /* cmd ::= KILL CONNECTION INTEGER */ +{setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} + break; + case 252: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ +{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} + break; + case 253: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ +{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: break; /********** End reduce actions ************************************************/ }; - assert( yyrulenostateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; } /* @@ -2885,7 +3385,8 @@ static void yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); @@ -2896,7 +3397,8 @@ static void yy_parse_failed( ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } #endif /* YYNOERRORRECOVERY */ @@ -2908,12 +3410,13 @@ static void yy_syntax_error( int yymajor, /* The major type of the error token */ ParseTOKENTYPE yyminor /* The minor type of the error token */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->pzErrMsg); + int32_t outputBufLen = tListLen(pInfo->msg); int32_t len = 0; if(TOKEN.z) { @@ -2923,18 +3426,19 @@ static void yy_syntax_error( if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { char tmpstr[128] = {0}; memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->pzErrMsg, msg, tmpstr); + len = sprintf(pInfo->msg, msg, tmpstr); } else { - len = sprintf(pInfo->pzErrMsg, msg, &TOKEN.z[0]); + len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); } } else { - len = sprintf(pInfo->pzErrMsg, "Incomplete SQL statement"); + len = sprintf(pInfo->msg, "Incomplete SQL statement"); } assert(len <= outputBufLen); /************ End %syntax_error code ******************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* @@ -2943,7 +3447,8 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); @@ -2958,7 +3463,8 @@ static void yy_accept( /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* The main parser program. @@ -2987,45 +3493,47 @@ void Parse( ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - unsigned int yyact; /* The parser action. */ + YYACTIONTYPE yyact; /* The parser action. */ #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ #endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif - yyParser *yypParser; /* The parser */ + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + ParseCTX_FETCH + ParseARG_STORE - yypParser = (yyParser*)yyp; assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif - ParseARG_STORE; + yyact = yypParser->yytos->stateno; #ifndef NDEBUG if( yyTraceFILE ){ - int stateno = yypParser->yytos->stateno; - if( stateno < YY_MIN_REDUCE ){ + if( yyact < YY_MIN_REDUCE ){ fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],stateno); + yyTracePrompt,yyTokenName[yymajor],yyact); }else{ fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE); + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); } } #endif do{ - yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); if( yyact >= YY_MIN_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor); + yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, + yyminor ParseCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,yymajor,yyminor); + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif - yymajor = YYNOCODE; + break; }else if( yyact==YY_ACCEPT_ACTION ){ yypParser->yytos--; yy_accept(yypParser); @@ -3076,10 +3584,9 @@ void Parse( yymajor = YYNOCODE; }else{ while( yypParser->yytos >= yypParser->yystack - && yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yytos->stateno, - YYERRORSYMBOL)) >= YY_MIN_REDUCE + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE ){ yy_pop_parser_stack(yypParser); } @@ -3096,6 +3603,8 @@ void Parse( } yypParser->yyerrcnt = 3; yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; #elif defined(YYNOERRORRECOVERY) /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -3106,8 +3615,7 @@ void Parse( */ yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - + break; #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -3129,10 +3637,10 @@ void Parse( yypParser->yyerrcnt = -1; #endif } - yymajor = YYNOCODE; + break; #endif } - }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack ); + }while( yypParser->yytos>yypParser->yystack ); #ifndef NDEBUG if( yyTraceFILE ){ yyStackEntry *i; @@ -3147,3 +3655,17 @@ void Parse( #endif return; } + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int ParseFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index 99f03a7ff8185f8537ebb243e00bce77992db344..ce7b2f94a177576c8046b299bcb2e695fb5ead2d 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -3,7 +3,7 @@ #include #include -#include "qAst.h" +#include "texpr.h" #include "taosmsg.h" #include "tsdb.h" #include "tskiplist.h" @@ -471,7 +471,7 @@ tExprNode *createExpr1() { pRight->pVal = (tVariant*) calloc(1, sizeof(tVariant)); pRight->pVal->nType = TSDB_DATA_TYPE_INT; - pRight->pVal->i64Key = 12; + pRight->pVal->i64 = 12; auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode)); pRoot->nodeType = TSQL_NODE_EXPR; @@ -574,7 +574,7 @@ void exprSerializeTest1() { tVariant* v2 = p2->_node.pRight->pVal; ASSERT_EQ(v1->nType, v2->nType); - ASSERT_EQ(v1->i64Key, v2->i64Key); + ASSERT_EQ(v1->i64, v2->i64); ASSERT_EQ(p1->_node.hasPK, p2->_node.hasPK); tExprTreeDestroy(&p1, nullptr); diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index 4a5f7fbbbed18b0a2c2cd7fff7ad3101cfa11c41..3088d6f8078483c39ec3780250e8bfa2d613f218 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -4,42 +4,9 @@ #include #include "taos.h" -#include "tsdb.h" - -#include "tstoken.h" -#include "tutil.h" - #include "qHistogram.h" - -/* test validate the names for table/database */ -TEST(testCase, histogram_binary_search) { - SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); - - pHisto->numOfEntries = 10; - for (int32_t i = 0; i < 10; ++i) { - pHisto->elems[i].num = 1; - pHisto->elems[i].val = i; - } - - int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); - assert(idx == 1); - - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); - assert(idx == 9); - - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); - assert(idx == 10); - - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); - assert(idx == 0); - - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); - assert(idx == 4); - - free(pHisto); -} - -TEST(testCase, histogram_add) { +namespace { +void doHistogramAddTest() { SHistogramInfo* pHisto = NULL; /** @@ -99,6 +66,56 @@ TEST(testCase, histogram_add) { tHistogramDestroy(&pRes); free(res); } +void doHistogramRepeatTest() { + SHistogramInfo* pHisto = NULL; + struct timeval systemTime; + + gettimeofday(&systemTime, NULL); + int64_t st = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + + for (int32_t i = 0; i < 1000; ++i) { + tHistogramAdd(&pHisto, -24 + i); + // tHistogramPrint(pHisto); + } + + tHistogramDestroy(&pHisto); + +} +} + +/* test validate the names for table/database */ +TEST(testCase, histogram_binary_search) { + SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); + + pHisto->numOfEntries = 10; + for (int32_t i = 0; i < 10; ++i) { + pHisto->elems[i].num = 1; + pHisto->elems[i].val = i; + } + + int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + assert(idx == 1); + + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + assert(idx == 9); + + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + assert(idx == 10); + + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + assert(idx == 0); + + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + assert(idx == 4); + + free(pHisto); +} + +TEST(testCase, histogram_add) { + doHistogramAddTest(); + doHistogramRepeatTest(); +} TEST(testCase, heapsort) { // int32_t num = 20; diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index fa2b58a10bd044491a2fabbaee6180c37320ae35..f3e0d3e119259d0a7cc8a94a6b43f4c71558bf78 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -3,7 +3,7 @@ #include #include -#include "tsqlfunction.h" +#include "qAggMain.h" #include "tcompare.h" TEST(testCase, patternMatchTest) { diff --git a/src/query/tests/percentileTest.cpp b/src/query/tests/percentileTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..104bfb3c06a9613bafcd4e0b3f39af4f9d102b04 --- /dev/null +++ b/src/query/tests/percentileTest.cpp @@ -0,0 +1,254 @@ +#include +#include + +#include "qResultbuf.h" +#include "taos.h" +#include "taosdef.h" + +#include "qPercentile.h" + +namespace { +tMemBucket *createBigIntDataBucket(int32_t start, int32_t end) { + tMemBucket *pBucket = tMemBucketCreate(sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, start, end); + for (int32_t i = start; i <= end; ++i) { + int64_t val = i; + tMemBucketPut(pBucket, &val, 1); + } + + return pBucket; +} + +tMemBucket *createIntDataBucket(int32_t start, int32_t end) { + tMemBucket *pBucket = tMemBucketCreate(sizeof(int32_t), TSDB_DATA_TYPE_INT, start, end); + + for (int32_t i = start; i <= end; ++i) { + int32_t val = i; + tMemBucketPut(pBucket, &val, 1); + } + + return pBucket; +} + +tMemBucket *createDoubleDataBucket(int32_t start, int32_t end) { + tMemBucket *pBucket = tMemBucketCreate(sizeof(double), TSDB_DATA_TYPE_DOUBLE, start, end); + for (int32_t i = start; i <= end; ++i) { + double val = i; + int32_t ret = tMemBucketPut(pBucket, &val, 1); + if (ret != 0) { + printf("value out of range:%f", val); + } + } + + return pBucket; +} + +tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) { + tMemBucket *pBucket = tMemBucketCreate(tDataTypes[type].bytes, type, start, end); + for (int32_t i = start; i <= end; ++i) { + uint64_t k = i; + int32_t ret = tMemBucketPut(pBucket, &k, 1); + if (ret != 0) { + printf("value out of range:%" PRId64, k); + } + } + + return pBucket; +} + +void intDataTest() { + printf("running %s\n", __FUNCTION__); + + tMemBucket *pBucket = NULL; + double result = 0.; + + pBucket = createIntDataBucket(0, 0); + result = getPercentile(pBucket, 0); + ASSERT_DOUBLE_EQ(result, 0); + tMemBucketDestroy(pBucket); + + pBucket = createIntDataBucket(0, 1); + result = getPercentile(pBucket, 100); + ASSERT_DOUBLE_EQ(result, 1); + + result = getPercentile(pBucket, 0); + ASSERT_DOUBLE_EQ(result, 0); + tMemBucketDestroy(pBucket); + + pBucket = createIntDataBucket(-1, 1); + + result = getPercentile(pBucket, 50); + ASSERT_DOUBLE_EQ(result, 0); + + result = getPercentile(pBucket, 0); + ASSERT_DOUBLE_EQ(result, -1); + + result = getPercentile(pBucket, 75); + ASSERT_DOUBLE_EQ(result, 0.5); + + result = getPercentile(pBucket, 100); + ASSERT_DOUBLE_EQ(result, 1); + tMemBucketDestroy(pBucket); + + pBucket = createIntDataBucket(0, 99999); + result = getPercentile(pBucket, 50); + ASSERT_DOUBLE_EQ(result, 49999.5); + + tMemBucketDestroy(pBucket); +} + +void bigintDataTest() { + printf("running %s\n", __FUNCTION__); + + tMemBucket *pBucket = NULL; + double result = 0.0; + + pBucket = createBigIntDataBucket(-1000, 1000); + result = getPercentile(pBucket, 50); + ASSERT_DOUBLE_EQ(result, 0.); + tMemBucketDestroy(pBucket); + + pBucket = createBigIntDataBucket(-10000, 10000); + result = getPercentile(pBucket, 100); + ASSERT_DOUBLE_EQ(result, 10000.0); + tMemBucketDestroy(pBucket); + + pBucket = createBigIntDataBucket(-10000, 10000); + result = getPercentile(pBucket, 75); + ASSERT_DOUBLE_EQ(result, 5000.0); + + tMemBucketDestroy(pBucket); +} + +void doubleDataTest() { + printf("running %s\n", __FUNCTION__); + + tMemBucket *pBucket = NULL; + double result = 0; + + pBucket = createDoubleDataBucket(-10, 10); + result = getPercentile(pBucket, 0); + ASSERT_DOUBLE_EQ(result, -10.0); + + printf("result is: %lf\n", result); + tMemBucketDestroy(pBucket); + + pBucket = createDoubleDataBucket(-100000, 100000); + result = getPercentile(pBucket, 25); + ASSERT_DOUBLE_EQ(result, -50000); + + printf("result is: %lf\n", result); + + tMemBucketDestroy(pBucket); + + pBucket = createDoubleDataBucket(-100000, 100000); + result = getPercentile(pBucket, 50); + ASSERT_DOUBLE_EQ(result, 0); + + tMemBucketDestroy(pBucket); + + pBucket = createDoubleDataBucket(-100000, 100000); + result = getPercentile(pBucket, 75); + ASSERT_DOUBLE_EQ(result, 50000); + tMemBucketDestroy(pBucket); + + pBucket = createDoubleDataBucket(-100000, 100000); + + result = getPercentile(pBucket, 100); + ASSERT_DOUBLE_EQ(result, 100000.0); + + printf("result is: %lf\n", result); + tMemBucketDestroy(pBucket); +} + +/* + * large data test, we employ 0.1billion double data to calculated the percentile + * which is 800MB data + */ +void largeDataTest() { + printf("running : %s\n", __FUNCTION__); + + tMemBucket *pBucket = NULL; + double result = 0; + + struct timeval tv; + gettimeofday(&tv, NULL); + + int64_t start = tv.tv_sec; + printf("start time: %" PRId64 "\n", tv.tv_sec); + pBucket = createDoubleDataBucket(0, 100000000); + result = getPercentile(pBucket, 50); + ASSERT_DOUBLE_EQ(result, 50000000); + + gettimeofday(&tv, NULL); + + printf("total elapsed time: %" PRId64 " sec.", -start + tv.tv_sec); + printf("the result of %d is: %lf\n", 50, result); + tMemBucketDestroy(pBucket); +} + +void qsortTest() { + printf("running : %s\n", __FUNCTION__); + + SSchema field[1] = { + {TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)}, + }; + + const int32_t num = 2000; + + int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num); + for (int32_t i = 0; i < num; ++i) { + d[i] = i % 4; + } + + const int32_t numOfOrderCols = 1; + int32_t orderColIdx = 0; + SColumnModel * pModel = createColumnModel(field, 1, 1000); + tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1); + + tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1); + + for (int32_t i = 0; i < num; ++i) { + printf("%d\t", d[i]); + } + printf("\n"); + + destroyColumnModel(pModel); +} + +void unsignedDataTest() { + printf("running %s\n", __FUNCTION__); + + tMemBucket *pBucket = NULL; + double result = 0.0; + + pBucket = createUnsignedDataBucket(0, 1000, TSDB_DATA_TYPE_UINT); + result = getPercentile(pBucket, 50); + ASSERT_DOUBLE_EQ(result, 500.0); + tMemBucketDestroy(pBucket); + + pBucket = createUnsignedDataBucket(0, 10000, TSDB_DATA_TYPE_UBIGINT); + result = getPercentile(pBucket, 100); + ASSERT_DOUBLE_EQ(result, 10000.0); + + result = getPercentile(pBucket, 0); + ASSERT_DOUBLE_EQ(result, 0.0); + + result = getPercentile(pBucket, 50); + ASSERT_DOUBLE_EQ(result, 5000); + + result = getPercentile(pBucket, 75); + ASSERT_DOUBLE_EQ(result, 7500); + tMemBucketDestroy(pBucket); + +} + +} // namespace + +TEST(testCase, percentileTest) { +// qsortTest(); + intDataTest(); + bigintDataTest(); + doubleDataTest(); + unsignedDataTest(); + largeDataTest(); +} diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index bada3194bd90e30de465dceaef07742cf6e8ac07..8ca636b8348cd08eddd98d5c1c24de15d4eaa2b4 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -34,7 +34,7 @@ void simpleTest() { int32_t num = 10; tVariant t = {0}; t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64Key = 1; + t.i64 = 1; int64_t* list = createTsList(10, 10000000, 30); tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); @@ -61,7 +61,7 @@ void largeTSTest() { int32_t num = 1000000; tVariant t = {0}; t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64Key = 1; + t.i64 = 1; int64_t* list = createTsList(num, 10000000, 30); tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); @@ -93,7 +93,7 @@ void multiTagsTest() { for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - t.i64Key = i; + t.i64 = i; tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -104,7 +104,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); EXPECT_EQ(pTSBuf->numOfGroups, 1); tsBufFlush(pTSBuf); @@ -131,7 +131,7 @@ void multiVnodeTagsTest() { for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - t.i64Key = i; + t.i64 = i; tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -144,11 +144,11 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -175,7 +175,7 @@ void loadDataTest() { for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - t.i64Key = i; + t.i64 = i; tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %" PRIu64 "\n", i, list[0]); @@ -190,11 +190,11 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -253,7 +253,7 @@ void TSTraverse() { for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - t.i64Key = i; + t.i64 = i; tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); @@ -297,14 +297,14 @@ void TSTraverse() { tVariant t = {0}; t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64Key = startTag; + t.i64 = startTag; tsBufGetElemStartPos(pTSBuf, startVnode, &t); int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -314,7 +314,7 @@ void TSTraverse() { totalOutput = 10; startTag -= 1; - t.i64Key = startTag; + t.i64 = startTag; tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag == 0) { @@ -345,14 +345,14 @@ void TSTraverse() { startVnode = 1; startTag = 2; - t.i64Key = startTag; + t.i64 = startTag; tsBufGetElemStartPos(pTSBuf, startVnode, &t); totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -362,7 +362,7 @@ void TSTraverse() { totalOutput = 10; startTag -= 1; - t.i64Key = startTag; + t.i64 = startTag; tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag < 0) { @@ -414,7 +414,7 @@ void mergeDiffVnodeBufferTest() { int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - t.i64Key = i; + t.i64 = i; tsBufAppend(pTSBuf1, 1, &t, (const char*)list, num * sizeof(int64_t)); tsBufAppend(pTSBuf2, 9, &t, (const char*)list, num * sizeof(int64_t)); @@ -451,7 +451,7 @@ void mergeIdenticalVnodeBufferTest() { int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - t.i64Key = i; + t.i64 = i; tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -462,7 +462,7 @@ void mergeIdenticalVnodeBufferTest() { for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); - t.i64Key = i; + t.i64 = i; tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -487,7 +487,7 @@ void mergeIdenticalVnodeBufferTest() { EXPECT_EQ(elem.id, 77); } - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); } tsBufDestroy(pTSBuf1); diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index b6e1170d0ef989a7fe4bad43a29ccfb357005883..3406d8309023118d16cfc4dfbdab58defc6f005a 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -27,22 +27,22 @@ int32_t testValidateName(char* name) { } static void _init_tvariant_bool(tVariant* t) { - t->i64Key = TSDB_FALSE; + t->i64 = TSDB_FALSE; t->nType = TSDB_DATA_TYPE_BOOL; } static void _init_tvariant_tinyint(tVariant* t) { - t->i64Key = -27; + t->i64 = -27; t->nType = TSDB_DATA_TYPE_TINYINT; } static void _init_tvariant_int(tVariant* t) { - t->i64Key = -23997659; + t->i64 = -23997659; t->nType = TSDB_DATA_TYPE_INT; } static void _init_tvariant_bigint(tVariant* t) { - t->i64Key = -3333333333333; + t->i64 = -3333333333333; t->nType = TSDB_DATA_TYPE_BIGINT; } @@ -227,10 +227,10 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_SQL); char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_SQL); char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_SQL); char t62[] = " ABC . def "; EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_SQL); @@ -249,13 +249,13 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_SQL); char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_SQL); char t67[] = "abc . ' DEF '"; EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_SQL); char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_SQL); // do not use key words char t69[] = "table.'DEF'"; @@ -265,7 +265,7 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_SQL); char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_SQL); char t72[] = "'_abDEF&^%1234'.' DIef'"; EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_SQL); @@ -446,19 +446,19 @@ TEST(testCase, tvariant_convert) { _init_tvariant_bool(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64Key, 0); + EXPECT_EQ(t.i64, 0); _init_tvariant_bool(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i64Key, 0); + EXPECT_EQ(t.i64, 0); _init_tvariant_bool(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i64Key, 0); + EXPECT_EQ(t.i64, 0); _init_tvariant_bool(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64Key, 0); + EXPECT_EQ(t.i64, 0); _init_tvariant_bool(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); @@ -481,23 +481,23 @@ TEST(testCase, tvariant_convert) { // 2. tinyint to other data types _init_tvariant_tinyint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64Key, 1); + EXPECT_EQ(t.i64, 1); _init_tvariant_tinyint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i64Key, -27); + EXPECT_EQ(t.i64, -27); _init_tvariant_tinyint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i64Key, -27); + EXPECT_EQ(t.i64, -27); _init_tvariant_tinyint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i64Key, -27); + EXPECT_EQ(t.i64, -27); _init_tvariant_tinyint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64Key, -27); + EXPECT_EQ(t.i64, -27); _init_tvariant_tinyint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); @@ -521,7 +521,7 @@ TEST(testCase, tvariant_convert) { // types////////////////////////////////////////////////////////////////// _init_tvariant_int(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64Key, 1); + EXPECT_EQ(t.i64, 1); _init_tvariant_int(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); @@ -531,11 +531,11 @@ TEST(testCase, tvariant_convert) { _init_tvariant_int(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i64Key, -23997659); + EXPECT_EQ(t.i64, -23997659); _init_tvariant_int(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64Key, -23997659); + EXPECT_EQ(t.i64, -23997659); _init_tvariant_int(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); @@ -559,7 +559,7 @@ TEST(testCase, tvariant_convert) { // type////////////////////////////////////////////////////////////////////////////// _init_tvariant_bigint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64Key, 1); + EXPECT_EQ(t.i64, 1); _init_tvariant_bigint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); @@ -572,7 +572,7 @@ TEST(testCase, tvariant_convert) { _init_tvariant_bigint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64Key, -3333333333333); + EXPECT_EQ(t.i64, -3333333333333); _init_tvariant_bigint(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); @@ -596,11 +596,11 @@ TEST(testCase, tvariant_convert) { // types//////////////////////////////////////////////////////////////////////// _init_tvariant_float(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64Key, 1); + EXPECT_EQ(t.i64, 1); _init_tvariant_float(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64Key, -8991212199); + EXPECT_EQ(t.i64, -8991212199); _init_tvariant_float(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); @@ -626,14 +626,14 @@ TEST(testCase, tvariant_convert) { t.nLen = strlen(t.pz); t.nType = TSDB_DATA_TYPE_BINARY; EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64Key, 1); + EXPECT_EQ(t.i64, 1); _init_tvariant_binary(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); _init_tvariant_binary(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64Key, 200000); + EXPECT_EQ(t.i64, 200000); _init_tvariant_binary(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); @@ -659,14 +659,14 @@ TEST(testCase, tvariant_convert) { t.nLen = wcslen(t.wpz); t.nType = TSDB_DATA_TYPE_NCHAR; EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64Key, 0); + EXPECT_EQ(t.i64, 0); _init_tvariant_nchar(&t); EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); _init_tvariant_nchar(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64Key, -2000000); + EXPECT_EQ(t.i64, -2000000); _init_tvariant_nchar(&t); EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); @@ -732,34 +732,34 @@ static SStrToken createStrToken(char* s) { TEST(testCase, isValidNumber_test) { SStrToken t1 = createStrToken("123abc"); - EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); t1 = createStrToken("0xabc"); - EXPECT_EQ(isValidNumber(&t1), TK_HEX); + EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); t1 = createStrToken("0b11101"); - EXPECT_EQ(isValidNumber(&t1), TK_BIN); + EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); t1 = createStrToken(".134abc"); - EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); t1 = createStrToken("1e1 "); - EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); t1 = createStrToken("1+2"); - EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); t1 = createStrToken("-0x123"); - EXPECT_EQ(isValidNumber(&t1), TK_HEX); + EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); t1 = createStrToken("-1"); - EXPECT_EQ(isValidNumber(&t1), TK_INTEGER); + EXPECT_EQ(tGetNumericStringType(&t1), TK_INTEGER); t1 = createStrToken("-0b1110"); - EXPECT_EQ(isValidNumber(&t1), TK_BIN); + EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); t1 = createStrToken("-.234"); - EXPECT_EQ(isValidNumber(&t1), TK_FLOAT); + EXPECT_EQ(tGetNumericStringType(&t1), TK_FLOAT); } TEST(testCase, getTempFilePath_test) { diff --git a/src/rpc/inc/rpcHead.h b/src/rpc/inc/rpcHead.h index 520edadc7dd072849720cad53c7f6f4ba605a06c..5b401ac54be194a38bad2f7680155f5edb8350b4 100644 --- a/src/rpc/inc/rpcHead.h +++ b/src/rpc/inc/rpcHead.h @@ -20,10 +20,6 @@ extern "C" { #endif -#define RPC_CONN_UDPS 0 -#define RPC_CONN_UDPC 1 -#define RPC_CONN_TCPS 2 -#define RPC_CONN_TCPC 3 #define RPC_CONN_TCP 2 extern int tsRpcOverhead; @@ -58,6 +54,7 @@ typedef struct { char empty[1]; // reserved uint8_t msgType; // message type int32_t msgLen; // message length including the header iteslf + uint32_t msgVer; int32_t code; // code in response message uint8_t content[0]; // message body starts from here } SRpcHead; diff --git a/src/rpc/inc/rpcLog.h b/src/rpc/inc/rpcLog.h index 10e847669146251ff5b3b4a5185bd09625f6ba07..6c4a281d2c0f0bf85975200c833acbac856dc40c 100644 --- a/src/rpc/inc/rpcLog.h +++ b/src/rpc/inc/rpcLog.h @@ -23,7 +23,7 @@ extern "C" { #include "tlog.h" extern int32_t rpcDebugFlag; -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; #define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 09d8f3bff1faad5596d85e931adea7f83670a48a..60a12c26b78626ed81cbc182d76c836c6ee74498 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -272,7 +272,7 @@ static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) } static void rpcLockCache(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { @@ -282,7 +282,7 @@ static void rpcLockCache(int64_t *lockedBy) { } static void rpcUnlockCache(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { assert(false); } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 00a97d7bc2474e31931e07edfdc4df617578e531..cae227cbdb7363bc0b26c80f1cc9edcbf1574d71 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -142,7 +142,6 @@ static int32_t tsRpcNum = 0; #define RPC_CONN_UDPC 1 #define RPC_CONN_TCPS 2 #define RPC_CONN_TCPC 3 -#define RPC_CONN_TCP 2 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { taosInitUdpConnection, @@ -247,8 +246,12 @@ void *rpcOpen(const SRpcInit *pInit) { if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); pRpc->connType = pInit->connType; + if (pRpc->connType == TAOS_CONN_CLIENT) { + pRpc->numOfThreads = pInit->numOfThreads; + } else { + pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; + } pRpc->idleTime = pInit->idleTime; - pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; pRpc->localPort = pInit->localPort; pRpc->afp = pInit->afp; pRpc->sessions = pInit->sessions+1; @@ -313,7 +316,7 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } - tDebug("%s rpc is openned, threads:%d sessions:%d", pRpc->label, pRpc->numOfThreads, pInit->sessions); + tDebug("%s rpc is opened, threads:%d sessions:%d", pRpc->label, pRpc->numOfThreads, pInit->sessions); return pRpc; } @@ -404,7 +407,7 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64 if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META - || type == TSDB_MSG_TYPE_CM_SHOW ) + || type == TSDB_MSG_TYPE_CM_SHOW || type == TSDB_MSG_TYPE_DM_STATUS) pContext->connType = RPC_CONN_TCPC; pContext->rid = taosAddRef(tsRpcRefId, pContext); @@ -576,7 +579,7 @@ static void rpcFreeMsg(void *msg) { static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { SRpcConn *pConn; - uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); + uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -631,15 +634,19 @@ static void rpcReleaseConn(SRpcConn *pConn) { // if there is an outgoing message, free it if (pConn->outType && pConn->pReqMsg) { SRpcReqContext *pContext = pConn->pContext; - if (pContext->pRsp) { + if (pContext) { + if (pContext->pRsp) { // for synchronous API, post semaphore to unblock app - pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; - pContext->pRsp->pCont = NULL; - pContext->pRsp->contLen = 0; - tsem_post(pContext->pSem); + pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; + pContext->pRsp->pCont = NULL; + pContext->pRsp->contLen = 0; + tsem_post(pContext->pSem); + } + pContext->pConn = NULL; + taosRemoveRef(tsRpcRefId, pContext->rid); + } else { + assert(0); } - pContext->pConn = NULL; - taosRemoveRef(tsRpcRefId, pContext->rid); } } @@ -955,6 +962,11 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL; } + if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { + tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, taosMsg[pHead->msgType]); + terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL; + } + pConn = rpcGetConnObj(pRpc, sid, pRecv); if (pConn == NULL) { tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); @@ -1083,16 +1095,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { rpcCloseConn(pConn); } - tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); + if (pHead->msgType + 1 > 1 && pHead->msgType+1 < TSDB_MSG_TYPE_MAX) { + tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); + } else { + tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], code); + } } } else { // msg is passed to app only parsing is ok - - if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { - rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS); - rpcFreeMsg(pRecv->msg); - return pConn; - } - rpcProcessIncomingMsg(pConn, pHead, pContext); } } @@ -1211,6 +1220,7 @@ static void rpcSendReqHead(SRpcConn *pConn) { pHead = (SRpcHead *)msg; pHead->version = 1; pHead->msgType = pConn->outType; + pHead->msgVer = htonl(tsVersion >> 8); pHead->spi = pConn->spi; pHead->encrypt = 0; pHead->tranId = pConn->outTranId; @@ -1271,7 +1281,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { SRpcConn *pConn = rpcSetupConnToServer(pContext); if (pConn == NULL) { pContext->code = terrno; - taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); + taosTmrStart(rpcProcessConnError, 1, pContext, pRpc->tmrCtrl); return; } @@ -1281,6 +1291,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { // set the message header pHead->version = 1; + pHead->msgVer = htonl(tsVersion >> 8); pHead->msgType = msgType; pHead->encrypt = 0; pConn->tranId++; @@ -1383,7 +1394,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pContext->pConn = NULL; pConn->pReqMsg = NULL; - taosTmrStart(rpcProcessConnError, 0, pConn->pContext, pRpc->tmrCtrl); + taosTmrStart(rpcProcessConnError, 1, pConn->pContext, pRpc->tmrCtrl); rpcReleaseConn(pConn); } } @@ -1560,6 +1571,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_INVALID_USER || code == TSDB_CODE_RPC_NOT_READY) { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); @@ -1597,7 +1609,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { } static void rpcLockConn(SRpcConn *pConn) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(&(pConn->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { @@ -1607,7 +1619,7 @@ static void rpcLockConn(SRpcConn *pConn) { } static void rpcUnlockConn(SRpcConn *pConn) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(pConn->lockedBy), tid, 0) != tid) { assert(false); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 2850046d050cba4adceeac615c0672d36dfed7d9..3162ab2e4cbabc2a1ce9ed863589e3be0e92f75f 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -17,25 +17,18 @@ #include "tsocket.h" #include "tutil.h" #include "taosdef.h" -#include "taoserror.h" +#include "taoserror.h" #include "rpcLog.h" #include "rpcHead.h" #include "rpcTcp.h" -#ifdef WINDOWS -#include "wepoll.h" -#endif - -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif typedef struct SFdObj { void *signature; SOCKET fd; // TCP socket FD - int closedByApp; // 1: already closed by App void *thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; + int16_t closedByApp; // 1: already closed by App struct SThreadObj *pThreadObj; struct SFdObj *prev; struct SFdObj *next; @@ -47,7 +40,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - SOCKET pollFd; + EpollFd pollFd; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -55,10 +48,19 @@ typedef struct SThreadObj { void *(*processData)(SRecvInfo *pPacket); } SThreadObj; +typedef struct { + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SThreadObj **pThreadObj; +} SClientObj; + typedef struct { SOCKET fd; uint32_t ip; uint16_t port; + int8_t stop; + int8_t reserve; char label[TSDB_LABEL_LEN]; int numOfThreads; void * shandle; @@ -79,7 +81,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); if (pServerObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -93,7 +95,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); free(pServerObj); return NULL; } @@ -103,24 +105,25 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - // initialize parameters in case it may encounter error later + // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); for (int j=0; jpThreadObj[j]); free(pServerObj->pThreadObj); free(pServerObj); return NULL; } - + pServerObj->pThreadObj[i] = pThreadObj; pThreadObj->pollFd = -1; taosResetPthread(&pThreadObj->thread); pThreadObj->processData = fp; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); pThreadObj->shandle = shandle; + pThreadObj->stop = false; } // initialize mutex, thread, fd which may fail @@ -132,7 +135,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -149,17 +152,17 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); - if (pServerObj->fd < 0) code = -1; + if (pServerObj->fd < 0) code = -1; - if (code == 0) { + if (code == 0) { code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); if (code != 0) { - tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); + tError("%s failed to create TCP accept thread(%s)", label, strerror(code)); } } if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); taosCleanUpTcpServer(pServerObj); pServerObj = NULL; } else { @@ -171,8 +174,9 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } static void taosStopTcpThread(SThreadObj* pThreadObj) { - // save thread into local variable and signal thread to stop - pthread_t thread = pThreadObj->thread; + if (pThreadObj == NULL) { return;} + // save thread into local variable and signal thread to stop + pthread_t thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { return; } @@ -188,8 +192,20 @@ void taosStopTcpServer(void *handle) { SServerObj *pServerObj = handle; if (pServerObj == NULL) return; - if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); + pServerObj->stop = 1; + if (pServerObj->fd >= 0) { +#ifdef WINDOWS + closesocket(pServerObj->fd); +#elif defined(__APPLE__) + if (pServerObj->fd!=-1) { + close(pServerObj->fd); + pServerObj->fd = -1; + } +#else + shutdown(pServerObj->fd, SHUT_RD); +#endif + } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { pthread_detach(pthread_self()); @@ -230,6 +246,11 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + if (pServerObj->stop) { + tDebug("%s TCP server stop accepting new connections", pServerObj->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label); @@ -241,8 +262,15 @@ static void *taosAcceptTcpConnection(void *arg) { } taosKeepTcpAlive(connFd); - struct timeval to={1, 0}; - taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + struct timeval to={5, 0}; + int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + if (ret != 0) { + taosCloseSocket(connFd); + tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), + taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); + continue; + } + // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; @@ -251,13 +279,13 @@ static void *taosAcceptTcpConnection(void *arg) { if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, + tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); - } + } // pick up next thread for next connection threadId++; @@ -268,70 +296,107 @@ static void *taosAcceptTcpConnection(void *arg) { return NULL; } -void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle) { - SThreadObj *pThreadObj; - pthread_attr_t thattr; - - pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj)); - memset(pThreadObj, 0, sizeof(SThreadObj)); - tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); - pThreadObj->ip = ip; - pThreadObj->shandle = shandle; - - if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) { - tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); - free(pThreadObj); - terrno = TAOS_SYSTEM_ERROR(errno); +void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { + SClientObj *pClientObj = (SClientObj *)calloc(1, sizeof(SClientObj)); + if (pClientObj == NULL) { + tError("TCP:%s no enough memory", label); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter - if (pThreadObj->pollFd < 0) { - tError("%s failed to create TCP client epoll", label); - free(pThreadObj); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; + tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); + pClientObj->numOfThreads = numOfThreads; + pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*)); + if (pClientObj->pThreadObj == NULL) { + tError("TCP:%s no enough memory", label); + tfree(pClientObj); + terrno = TAOS_SYSTEM_ERROR(errno); } - pThreadObj->processData = fp; - + int code = 0; + pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); - pthread_attr_destroy(&thattr); - if (code != 0) { - taosCloseSocket(pThreadObj->pollFd); - free(pThreadObj); - terrno = TAOS_SYSTEM_ERROR(errno); - tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); - return NULL; + + for (int i = 0; i < numOfThreads; ++i) { + SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); + if (pThreadObj == NULL) { + tError("TCP:%s no enough memory", label); + terrno = TAOS_SYSTEM_ERROR(errno); + for (int j=0; jpThreadObj[j]); + free(pClientObj); + pthread_attr_destroy(&thattr); + return NULL; + } + pClientObj->pThreadObj[i] = pThreadObj; + taosResetPthread(&pThreadObj->thread); + pThreadObj->ip = ip; + pThreadObj->stop = false; + tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); + pThreadObj->shandle = shandle; + pThreadObj->processData = fp; } - tDebug("%s TCP client is initialized, ip:%u:%hu", label, ip, port); + // initialize mutex, thread, fd which may fail + for (int i = 0; i < numOfThreads; ++i) { + SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; + code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + if (code < 0) { + tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); + break; + } + + pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + if (pThreadObj->pollFd < 0) { + tError("%s failed to create TCP epoll", label); + code = -1; + break; + } - return pThreadObj; + code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + if (code != 0) { + tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); + break; + } + pThreadObj->threadId = i; + } + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosCleanUpTcpClient(pClientObj); + pClientObj = NULL; + } + return pClientObj; } void taosStopTcpClient(void *chandle) { - SThreadObj *pThreadObj = chandle; - if (pThreadObj == NULL) return; + SClientObj *pClientObj = chandle; - tDebug ("%s TCP client is stopped", pThreadObj->label); + if (pClientObj == NULL) return; + + tDebug ("%s TCP client is stopped", pClientObj->label); } void taosCleanUpTcpClient(void *chandle) { - SThreadObj *pThreadObj = chandle; - if (pThreadObj == NULL) return; + SClientObj *pClientObj = chandle; + if (pClientObj == NULL) return; + for (int i = 0; i < pClientObj->numOfThreads; ++i) { + SThreadObj *pThreadObj= pClientObj->pThreadObj[i]; + taosStopTcpThread(pThreadObj); + } - tDebug ("%s TCP client will be cleaned up", pThreadObj->label); - taosStopTcpThread(pThreadObj); + tDebug("%s TCP client is cleaned up", pClientObj->label); + tfree(pClientObj->pThreadObj); + tfree(pClientObj); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { - SThreadObj * pThreadObj = shandle; + SClientObj * pClientObj = shandle; + int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; + atomic_store_32(&pClientObj->index, index + 1); + SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); - if (fd < 0) return NULL; + if (fd <= 0) return NULL; struct sockaddr_in sin; uint16_t localPort = 0; @@ -342,12 +407,12 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin } SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); - + if (pFdObj) { pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", + tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); @@ -362,7 +427,7 @@ void taosCloseTcpConnection(void *chandle) { if (pFdObj == NULL || pFdObj->signature != pFdObj) return; SThreadObj *pThreadObj = pFdObj->pThreadObj; - tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); + tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); // pFdObj->thandle = NULL; pFdObj->closedByApp = 1; @@ -375,7 +440,7 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand SThreadObj *pThreadObj = pFdObj->pThreadObj; int ret = taosWriteMsg(pFdObj->fd, data, len); - tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); return ret; } @@ -398,7 +463,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { recvInfo.chandle = NULL; recvInfo.connType = RPC_CONN_TCP; (*(pThreadObj->processData))(&recvInfo); - } + } taosFreeFdObj(pFdObj); } @@ -413,7 +478,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); - return -1; + return -1; } msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); @@ -431,14 +496,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s %p read error, leftLen:%d retLen:%d FD:%p", + tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); free(buffer); return -1; } memcpy(msg, &rpcHead, sizeof(SRpcHead)); - + pInfo->msg = msg; pInfo->msgLen = msgLen; pInfo->ip = pFdObj->ip; @@ -449,7 +514,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { pInfo->connType = RPC_CONN_TCP; if (pFdObj->closedByApp) { - free(buffer); + free(buffer); return -1; } @@ -463,7 +528,10 @@ static void *taosProcessTcpData(void *param) { SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - + +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ while (1) { int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThreadObj->stop) { @@ -494,7 +562,7 @@ static void *taosProcessTcpData(void *param) { } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - shutdown(pFdObj->fd, SHUT_WR); + shutdown(pFdObj->fd, SHUT_WR); continue; } @@ -502,10 +570,13 @@ static void *taosProcessTcpData(void *param) { if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); } - if (pThreadObj->stop) break; + if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); + if (pThreadObj->pollFd >=0) { + EpollClose(pThreadObj->pollFd); + pThreadObj->pollFd = -1; + } while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; @@ -537,7 +608,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -571,7 +642,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s %p TCP thread:%d, number of FDs is negative!!!", + tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); if (pFdObj->prev) { @@ -586,7 +657,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 22301fcecc83fb1f4c9a29132b0311e05b2382e6..7a46dbe5c3e1238da93043f8dc97047f84ff72e8 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -15,7 +15,6 @@ #include "os.h" #include "tsocket.h" -#include "tsystem.h" #include "ttimer.h" #include "tutil.h" #include "taosdef.h" @@ -140,6 +139,7 @@ void taosStopUdpConnection(void *handle) { pConn = pSet->udpConn + i; if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); if (pConn->fd >=0) taosCloseSocket(pConn->fd); + pConn->fd = -1; } for (int i = 0; i < pSet->threads; ++i) { @@ -197,9 +197,16 @@ static void *taosRecvUdpData(void *param) { while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); - if(dataLen <= 0) { - tDebug("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno)); - break; + if (dataLen <= 0) { + tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen, + pConn->fd); + + // for windows usage, remote shutdown also returns - 1 in windows client + if (pConn->fd == -1) { + break; + } else { + continue; + } } port = ntohs(sourceAdd.sin_port); diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt index 383ce1b0f64cb218800211de3576af58056c6af1..c10cea6c9dd8c53ab8608c8a736795f2318059d8 100644 --- a/src/rpc/test/CMakeLists.txt +++ b/src/rpc/test/CMakeLists.txt @@ -16,3 +16,17 @@ IF (TD_LINUX) ADD_EXECUTABLE(rserver ${SERVER_SRC}) TARGET_LINK_LIBRARIES(rserver trpc) ENDIF () + +IF (TD_DARWIN) + LIST(APPEND CLIENT_SRC ./rclient.c) + ADD_EXECUTABLE(rclient ${CLIENT_SRC}) + TARGET_LINK_LIBRARIES(rclient trpc) + + LIST(APPEND SCLIENT_SRC ./rsclient.c) + ADD_EXECUTABLE(rsclient ${SCLIENT_SRC}) + TARGET_LINK_LIBRARIES(rsclient trpc) + + LIST(APPEND SERVER_SRC ./rserver.c) + ADD_EXECUTABLE(rserver ${SERVER_SRC}) + TARGET_LINK_LIBRARIES(rserver trpc) +ENDIF () diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 5721525adee3fc847a1ba2476ccb0995fb50a65c..faa6d40da398045e43dba3bbdbaf9ef6c7ccb1ff 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -188,7 +188,8 @@ int main(int argc, char *argv[]) { tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); - getchar(); + int ch = getchar(); + UNUSED(ch); taosCloseLog(); diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index 60271c771ca0a01bd449cb878fe2269759250fd3..82d0bbf520843f5418d1004f2fe7c1be756b7b6f 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -3,16 +3,14 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX) - LIST(REMOVE_ITEM SRC src/tarbitrator.c) - ADD_LIBRARY(sync ${SRC}) - TARGET_LINK_LIBRARIES(sync tutil pthread common) - LIST(APPEND BIN_SRC src/tarbitrator.c) - LIST(APPEND BIN_SRC src/taosTcpPool.c) - ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) - TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) +LIST(REMOVE_ITEM SRC src/syncArbitrator.c) +ADD_LIBRARY(sync ${SRC}) +TARGET_LINK_LIBRARIES(sync tutil pthread common) - #ADD_SUBDIRECTORY(test) -ENDIF () +LIST(APPEND BIN_SRC src/syncArbitrator.c) +LIST(APPEND BIN_SRC src/syncTcp.c) +ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) +TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) + +#ADD_SUBDIRECTORY(test) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 6d0c52284f7f2429ccfa20389ccd66b2447610b9..91613ae35107f0a667798e730f6021687e3ced5e 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -13,12 +13,14 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_SYNCINT_H -#define TDENGINE_SYNCINT_H +#ifndef TDENGINE_SYNC_INT_H +#define TDENGINE_SYNC_INT_H #ifdef __cplusplus extern "C" { #endif +#include "syncMsg.h" +#include "twal.h" #define sFatal(...) { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); }} #define sError(...) { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); }} @@ -27,83 +29,22 @@ extern "C" { #define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} #define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} -typedef enum { - TAOS_SMSG_SYNC_DATA = 1, - TAOS_SMSG_FORWARD = 2, - TAOS_SMSG_FORWARD_RSP = 3, - TAOS_SMSG_SYNC_REQ = 4, - TAOS_SMSG_SYNC_RSP = 5, - TAOS_SMSG_SYNC_MUST = 6, - TAOS_SMSG_STATUS = 7, - TAOS_SMSG_SYNC_DATA_RSP = 8, -} ESyncMsgType; +#define SYNC_TCP_THREADS 2 +#define SYNC_MAX_NUM 2 #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) #define SYNC_RECV_BUFFER_SIZE (5*1024*1024) -#define SYNC_FWD_TIMER 300 -#define SYNC_ROLE_TIMER 10000 -#define SYNC_WAIT_AFTER_CHOOSE_MASTER 3 + +#define SYNC_MAX_FWDS 512 +#define SYNC_FWD_TIMER 300 +#define SYNC_ROLE_TIMER 15000 // ms +#define SYNC_CHECK_INTERVAL 1000 // ms +#define SYNC_WAIT_AFTER_CHOOSE_MASTER 10 // ms #define nodeRole pNode->peerInfo[pNode->selfIndex]->role #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version #define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus -#pragma pack(push, 1) - -typedef struct { - char type; // msg type - char pversion; // protocol version - char reserved[6]; // not used - int32_t vgId; // vg ID - int32_t len; // content length, does not include head - // char cont[]; // message content starts from here -} SSyncHead; - -typedef struct { - SSyncHead syncHead; - uint16_t port; - char fqdn[TSDB_FQDN_LEN]; - int32_t sourceId; // only for arbitrator -} SFirstPkt; - -typedef struct { - int8_t sync; -} SFirstPktRsp; - -typedef struct { - int8_t role; - uint64_t version; -} SPeerStatus; - -typedef struct { - int8_t role; - int8_t ack; - int8_t type; - int8_t reserved[3]; - uint16_t tranId; - uint64_t version; - SPeerStatus peersStatus[]; -} SPeersStatus; - -typedef struct { - char name[TSDB_FILENAME_LEN]; - uint32_t magic; - uint32_t index; - uint64_t fversion; - int64_t size; -} SFileInfo; - -typedef struct { - int8_t sync; -} SFileAck; - -typedef struct { - uint64_t version; - int32_t code; -} SFwdRsp; - -#pragma pack(pop) - typedef struct { char * buffer; int32_t bufferSize; @@ -141,13 +82,15 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - int32_t syncFd; - int32_t peerFd; // forward FD + SOCKET syncFd; + SOCKET peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process + int32_t refCount; + int8_t isArb; + int64_t rid; void * timer; void * pConn; - int32_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -157,6 +100,7 @@ typedef struct SSyncNode { int8_t quorum; int8_t selfIndex; uint32_t vgId; + int32_t refCount; int64_t rid; SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator SSyncPeer * pMaster; @@ -164,14 +108,17 @@ typedef struct SSyncNode { SSyncFwds * pSyncFwds; // saved forward info if quorum >1 void * pFwdTimer; void * pRoleTimer; - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; + void * pTsdb; + FGetWalInfo getWalInfoFp; + FWriteToCache writeToCacheFp; FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; - FNotifyFileSynced notifyFileSynced; - FGetVersion getVersion; + FNotifyRole notifyRoleFp; + FNotifyFlowCtrl notifyFlowCtrlFp; + FStartSyncFile startSyncFileFp; + FStopSyncFile stopSyncFileFp; + FGetVersion getVersionFp; + FSendFile sendFileFp; + FRecvFile recvFileFp; pthread_mutex_t mutex; } SSyncNode; @@ -180,13 +127,13 @@ extern int32_t tsSyncNum; extern char tsNodeFqdn[TSDB_FQDN_LEN]; extern char * syncStatus[]; -void *syncRetrieveData(void *param); -void *syncRestoreData(void *param); -int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); -void syncRestartConnection(SSyncPeer *pPeer); -void syncBroadcastStatus(SSyncNode *pNode); -void syncAddPeerRef(SSyncPeer *pPeer); -int32_t syncDecPeerRef(SSyncPeer *pPeer); +void * syncRetrieveData(void *param); +void * syncRestoreData(void *param); +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); +void syncRestartConnection(SSyncPeer *pPeer); +void syncBroadcastStatus(SSyncNode *pNode); +SSyncPeer *syncAcquirePeer(int64_t rid); +void syncReleasePeer(SSyncPeer *pPeer); #ifdef __cplusplus } diff --git a/src/sync/inc/syncMsg.h b/src/sync/inc/syncMsg.h new file mode 100644 index 0000000000000000000000000000000000000000..85ac9c78affa5282d5ca703caffc1bc5c24461bb --- /dev/null +++ b/src/sync/inc/syncMsg.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_MSG_H +#define TDENGINE_SYNC_MSG_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "tsync.h" + +typedef enum { + TAOS_SMSG_START = 0, + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_SYNC_DATA_RSP = 2, + TAOS_SMSG_SYNC_FWD = 3, + TAOS_SMSG_SYNC_FWD_RSP = 4, + TAOS_SMSG_SYNC_REQ = 5, + TAOS_SMSG_SYNC_REQ_RSP = 6, + TAOS_SMSG_SYNC_MUST = 7, + TAOS_SMSG_SYNC_MUST_RSP = 8, + TAOS_SMSG_STATUS = 9, + TAOS_SMSG_STATUS_RSP = 10, + TAOS_SMSG_SETUP = 11, + TAOS_SMSG_SETUP_RSP = 12, + TAOS_SMSG_SYNC_FILE = 13, + TAOS_SMSG_SYNC_FILE_RSP = 14, + TAOS_SMSG_TEST = 15, + TAOS_SMSG_END = 16 +} ESyncMsgType; + +typedef enum { + SYNC_STATUS_BROADCAST, + SYNC_STATUS_BROADCAST_RSP, + SYNC_STATUS_SETUP_CONN, + SYNC_STATUS_SETUP_CONN_RSP, + SYNC_STATUS_EXCHANGE_DATA, + SYNC_STATUS_EXCHANGE_DATA_RSP, + SYNC_STATUS_CHECK_ROLE, + SYNC_STATUS_CHECK_ROLE_RSP +} ESyncStatusType; + +#pragma pack(push, 1) + +typedef struct { + int8_t type; // msg type + int8_t protocol; // protocol version + uint16_t signature; // fixed value + int32_t code; // + int32_t cId; // cluster Id + int32_t vgId; // vg ID + int32_t len; // content length, does not include head + uint32_t cksum; +} SSyncHead; + +typedef struct { + SSyncHead head; + uint16_t port; + uint16_t tranId; + int32_t sourceId; // only for arbitrator + char fqdn[TSDB_FQDN_LEN]; +} SSyncMsg; + +typedef struct { + SSyncHead head; + int8_t sync; + int8_t reserved; + uint16_t tranId; + int8_t reserverd[4]; +} SSyncRsp; + +typedef struct { + int8_t role; + uint64_t version; +} SPeerStatus; + +typedef struct { + SSyncHead head; + int8_t role; + int8_t ack; + int8_t type; + int8_t reserved[3]; + uint16_t tranId; + uint64_t version; + SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; +} SPeersStatus; + +typedef struct { + SSyncHead head; + uint64_t fversion; +} SFileVersion; + +typedef struct { + SSyncHead head; + int8_t ack; +} SFileAck; + +typedef struct { + SSyncHead head; + uint64_t version; + int32_t code; +} SFwdRsp; + +#pragma pack(pop) + +#define SYNC_PROTOCOL_VERSION 1 +#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) + +extern char *statusType[]; + +uint16_t syncGenTranId(); +int32_t syncCheckHead(SSyncHead *pHead); + +void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); +void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); + +void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); +void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_VNODEPEER_H diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/syncTcp.h similarity index 65% rename from src/sync/inc/taosTcpPool.h rename to src/sync/inc/syncTcp.h index 41043b0cd4c886616d5cecd2739eae684052c395..b322c3440c4eeb5b53b1d55a2446030b4f892e0b 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/syncTcp.h @@ -13,30 +13,27 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TCP_POOL_H -#define TDENGINE_TCP_POOL_H +#ifndef TDENGINE_SYNC_TCP_POOL_H +#define TDENGINE_SYNC_TCP_POOL_H #ifdef __cplusplus extern "C" { #endif -typedef void *ttpool_h; -typedef void *tthread_h; - typedef struct { int32_t numOfThreads; uint32_t serverIp; int16_t port; int32_t bufferSize; - void (*processBrokenLink)(void *ahandle); - int32_t (*processIncomingMsg)(void *ahandle, void *buffer); - void (*processIncomingConn)(int32_t fd, uint32_t ip); + void (*processBrokenLink)(int64_t handleId); + int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); + void (*processIncomingConn)(SOCKET fd, uint32_t ip); } SPoolInfo; -ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo); -void taosCloseTcpThreadPool(ttpool_h); -void * taosAllocateTcpConn(void *, void *ahandle, int32_t connFd); -void taosFreeTcpConn(void *); +void *syncOpenTcpThreadPool(SPoolInfo *pInfo); +void syncCloseTcpThreadPool(void *); +void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); +void syncFreeTcpConn(void *); #ifdef __cplusplus } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/syncArbitrator.c similarity index 69% rename from src/sync/src/tarbitrator.c rename to src/sync/src/syncArbitrator.c index 4016042de2135f732dfeb2bbc3b0fdc65b1b63f6..fdbef4c9f827d77bdc1ce1874dc2b368a7467ca1 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -22,21 +22,22 @@ #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" -#include "taosTcpPool.h" #include "twal.h" #include "tsync.h" #include "syncInt.h" +#include "syncTcp.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); -static void arbProcessBrokenLink(void *param); -static int32_t arbProcessPeerMsg(void *param, void *buffer); -static tsem_t tsArbSem; -static ttpool_h tsArbTcpPool; +extern void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd); +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); +static void arbProcessBrokenLink(int64_t rid); +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); +static tsem_t tsArbSem; +static void * tsArbTcpPool; typedef struct { char id[TSDB_EP_LEN + 24]; - int32_t nodeFd; + SOCKET nodeFd; void * pConn; } SNodeConn; @@ -69,12 +70,10 @@ int32_t main(int32_t argc, char *argv[]) { } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = arbSignalHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, arbSignalHandler); + taosSetSignal(SIGINT, arbSignalHandler); + taosSetSignal(SIGHUP, arbSignalHandler); + taosSetSignal(SIGABRT, arbSignalHandler); tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -90,7 +89,7 @@ int32_t main(int32_t argc, char *argv[]) { info.processBrokenLink = arbProcessBrokenLink; info.processIncomingMsg = arbProcessPeerMsg; info.processIncomingConn = arbProcessIncommingConnection; - tsArbTcpPool = taosOpenTcpThreadPool(&info); + tsArbTcpPool = syncOpenTcpThreadPool(&info); if (tsArbTcpPool == NULL) { sDebug("failed to open TCP thread pool, exit..."); @@ -101,25 +100,30 @@ int32_t main(int32_t argc, char *argv[]) { tsem_wait(&tsArbSem); - taosCloseTcpThreadPool(tsArbTcpPool); - sInfo("TAOS arbitrator is shut down\n"); - closelog(); + syncCloseTcpThreadPool(tsArbTcpPool); + sInfo("TAOS arbitrator is shut down"); + closelog(); return 0; } -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); - SFirstPkt firstPkt; - if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { - sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno)); + SSyncMsg msg; + if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sError("failed to read peer sync msg from ip:%s since %s", ipstr, strerror(errno)); taosCloseSocket(connFd); return; } + if (msg.head.type == TAOS_SMSG_TEST) { + syncProcessTestMsg(&msg, connFd); + return; + } + SNodeConn *pNode = calloc(sizeof(SNodeConn), 1); if (pNode == NULL) { sError("failed to allocate memory since %s", strerror(errno)); @@ -127,9 +131,9 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } - firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0; - snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); - if (firstPkt.syncHead.vgId) { + msg.fqdn[TSDB_FQDN_LEN - 1] = 0; + snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", msg.sourceId, msg.fqdn, msg.port); + if (msg.head.vgId) { sDebug("%s, vgId in head is not zero, close the connection", pNode->id); tfree(pNode); taosCloseSocket(connFd); @@ -138,26 +142,26 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { sDebug("%s, arbitrator request is accepted", pNode->id); pNode->nodeFd = connFd; - pNode->pConn = taosAllocateTcpConn(tsArbTcpPool, pNode, connFd); + pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd); return; } -static void arbProcessBrokenLink(void *param) { - SNodeConn *pNode = param; +static void arbProcessBrokenLink(int64_t rid) { + SNodeConn *pNode = (SNodeConn *)rid; sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno)); tfree(pNode); } -static int32_t arbProcessPeerMsg(void *param, void *buffer) { - SNodeConn *pNode = param; +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { + SNodeConn *pNode = (SNodeConn *)rid; SSyncHead head; int32_t bytes = 0; char * cont = (char *)buffer; - int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head)); - if (hlen != sizeof(head)) { + int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(SSyncHead)); + if (hlen != sizeof(SSyncHead)) { sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen); return -1; } @@ -172,14 +176,13 @@ static int32_t arbProcessPeerMsg(void *param, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context) { + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGHUP); - sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); + sInfo("shut down signal is %d", signum); // inform main thread to exit tsem_post(&tsArbSem); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index b73ca27ce977884638f7726e5c16916537234c2a..956ccdc0730e345d2e0ece3bcf45e4d0e9b3c14d 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -23,47 +23,46 @@ #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" -#include "taosTcpPool.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" +#include "syncTcp.h" #include "syncInt.h" -// global configurable -int32_t tsMaxSyncNum = 2; -int32_t tsSyncTcpThreads = 2; -int32_t tsMaxWatchFiles = 500; -int32_t tsMaxFwdInfo = 200; -int32_t tsSyncTimer = 1; +int32_t tsSyncNum = 0; // number of sync in process in whole system +char tsNodeFqdn[TSDB_FQDN_LEN] = {0}; -// module global, not configurable -int32_t tsSyncNum; // number of sync in process in whole system -char tsNodeFqdn[TSDB_FQDN_LEN]; - -static ttpool_h tsTcpPool; -static void * tsSyncTmrCtrl = NULL; -static void * tsVgIdHash; -static int32_t tsSyncRefId = -1; +static void * tsTcpPool = NULL; +static void * tsSyncTmrCtrl = NULL; +static void * tsVgIdHash = NULL; +static int32_t tsNodeRefId = -1; +static int32_t tsPeerRefId = -1; // local functions static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); static void syncRecoverFromMaster(SSyncPeer *pPeer); static void syncCheckPeerConnection(void *param, void *tmrId); -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); -static void syncProcessBrokenLink(void *param); -static int32_t syncProcessPeerMsg(void *param, void *buffer); -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); +static void syncProcessBrokenLink(int64_t rid); +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); +static void syncFreePeer(void *); static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); static void syncMonitorFwdInfos(void *param, void *tmrId); static void syncMonitorNodeRole(void *param, void *tmrId); static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); +static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); static void syncRestartPeer(SSyncPeer *pPeer); static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp); + static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); +static void syncStartCheckPeerConn(SSyncPeer *pPeer); +static void syncStopCheckPeerConn(SSyncPeer *pPeer); +static SSyncNode *syncAcquireNode(int64_t rid); +static void syncReleaseNode(SSyncNode *pNode); char* syncRole[] = { "offline", @@ -81,36 +80,10 @@ char *syncStatus[] = { "invalid" }; -typedef enum { - SYNC_STATUS_BROADCAST, - SYNC_STATUS_BROADCAST_RSP, - SYNC_STATUS_SETUP_CONN, - SYNC_STATUS_SETUP_CONN_RSP, - SYNC_STATUS_EXCHANGE_DATA, - SYNC_STATUS_EXCHANGE_DATA_RSP, - SYNC_STATUS_CHECK_ROLE, - SYNC_STATUS_CHECK_ROLE_RSP -} ESyncStatusType; - -char *statusType[] = { - "broadcast", - "broadcast-rsp", - "setup-conn", - "setup-conn-rsp", - "exchange-data", - "exchange-data-rsp", - "check-role", - "check-role-rsp" -}; - -uint16_t syncGenTranId() { - return taosRand() & 0XFFFF; -} - int32_t syncInit() { SPoolInfo info = {0}; - info.numOfThreads = tsSyncTcpThreads; + info.numOfThreads = SYNC_TCP_THREADS; info.serverIp = 0; info.port = tsSyncPort; info.bufferSize = SYNC_MAX_SIZE; @@ -118,32 +91,37 @@ int32_t syncInit() { info.processIncomingMsg = syncProcessPeerMsg; info.processIncomingConn = syncProcessIncommingConnection; - tsTcpPool = taosOpenTcpThreadPool(&info); + tsTcpPool = syncOpenTcpThreadPool(&info); if (tsTcpPool == NULL) { sError("failed to init tcpPool"); + syncCleanUp(); return -1; } tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); if (tsSyncTmrCtrl == NULL) { sError("failed to init tmrCtrl"); - taosCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; + syncCleanUp(); return -1; } tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVgIdHash == NULL) { - sError("failed to init tsVgIdHash"); - taosTmrCleanUp(tsSyncTmrCtrl); - taosCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; - tsSyncTmrCtrl = NULL; + sError("failed to init vgIdHash"); + syncCleanUp(); return -1; } - tsSyncRefId = taosOpenRef(200, syncFreeNode); - if (tsSyncRefId < 0) { + tsNodeRefId = taosOpenRef(200, syncFreeNode); + if (tsNodeRefId < 0) { + sError("failed to init node ref"); + syncCleanUp(); + return -1; + } + + tsPeerRefId = taosOpenRef(1000, syncFreePeer); + if (tsPeerRefId < 0) { + sError("failed to init peer ref"); syncCleanUp(); return -1; } @@ -155,12 +133,12 @@ int32_t syncInit() { } void syncCleanUp() { - if (tsTcpPool) { - taosCloseTcpThreadPool(tsTcpPool); + if (tsTcpPool != NULL) { + syncCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; } - if (tsSyncTmrCtrl) { + if (tsSyncTmrCtrl != NULL) { taosTmrCleanUp(tsSyncTmrCtrl); tsSyncTmrCtrl = NULL; } @@ -170,8 +148,15 @@ void syncCleanUp() { tsVgIdHash = NULL; } - taosCloseRef(tsSyncRefId); - tsSyncRefId = -1; + if (tsNodeRefId != -1) { + taosCloseRef(tsNodeRefId); + tsNodeRefId = -1; + } + + if (tsPeerRefId != -1) { + taosCloseRef(tsPeerRefId); + tsPeerRefId = -1; + } sInfo("sync module is cleaned up"); } @@ -189,22 +174,26 @@ int64_t syncStart(const SSyncInfo *pInfo) { tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); pthread_mutex_init(&pNode->mutex, NULL); - pNode->getFileInfo = pInfo->getFileInfo; - pNode->getWalInfo = pInfo->getWalInfo; - pNode->writeToCache = pInfo->writeToCache; - pNode->notifyRole = pInfo->notifyRole; + pNode->getWalInfoFp = pInfo->getWalInfoFp; + pNode->writeToCacheFp = pInfo->writeToCacheFp; + pNode->notifyRoleFp = pInfo->notifyRoleFp; pNode->confirmForward = pInfo->confirmForward; - pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl; - pNode->notifyFileSynced = pInfo->notifyFileSynced; - pNode->getVersion = pInfo->getVersion; + pNode->notifyFlowCtrlFp = pInfo->notifyFlowCtrlFp; + pNode->startSyncFileFp = pInfo->startSyncFileFp; + pNode->stopSyncFileFp = pInfo->stopSyncFileFp; + pNode->getVersionFp = pInfo->getVersionFp; + pNode->sendFileFp = pInfo->sendFileFp; + pNode->recvFileFp = pInfo->recvFileFp; pNode->selfIndex = -1; pNode->vgId = pInfo->vgId; pNode->replica = pCfg->replica; pNode->quorum = pCfg->quorum; + pNode->pTsdb = pInfo->pTsdb; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - pNode->rid = taosAddRef(tsSyncRefId, pNode); + pNode->refCount = 1; + pNode->rid = taosAddRef(tsNodeRefId, pNode); if (pNode->rid < 0) { syncFreeNode(pNode); return -1; @@ -226,7 +215,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } if (pNode->selfIndex < 0) { - sInfo("vgId:%d, this node is not configured", pNode->vgId); + sError("vgId:%d, this node is not configured", pNode->vgId); terrno = TSDB_CODE_SYN_INVALID_CONFIG; syncStop(pNode->rid); return -1; @@ -237,7 +226,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]); - pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo * sizeof(SFwdInfo), 1); + pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + SYNC_MAX_FWDS * sizeof(SFwdInfo), 1); if (pNode->pSyncFwds == NULL) { sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId); terrno = TAOS_SYSTEM_ERROR(errno); @@ -262,8 +251,13 @@ int64_t syncStart(const SSyncInfo *pInfo) { syncAddArbitrator(pNode); taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *)); - if (pNode->notifyRole) { - (*pNode->notifyRole)(pNode->vgId, nodeRole); + if (pNode->notifyRoleFp) { + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); + } + + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); } return pNode->rid; @@ -272,7 +266,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { void syncStop(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; sInfo("vgId:%d, cleanup sync", pNode->vgId); @@ -293,14 +287,14 @@ void syncStop(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); - taosRemoveRef(tsSyncRefId, rid); + syncReleaseNode(pNode); + taosRemoveRef(tsNodeRefId, rid); } int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { int32_t i, j; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, @@ -308,6 +302,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { pthread_mutex_lock(&pNode->mutex); + syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStopCheckPeerConn(pNode->peerInfo[index]); + } + for (i = 0; i < pNode->replica; ++i) { for (j = 0; j < pNewCfg->replica; ++j) { if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && @@ -337,6 +336,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { newPeers[i] = pNode->peerInfo[j]; } + if (newPeers[i] == NULL) { + sError("vgId:%d, failed to reconfig", pNode->vgId); + return TSDB_CODE_SYN_INVALID_CONFIG; + } + if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) { pNode->selfIndex = i; } @@ -356,7 +360,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { if (pNewCfg->replica <= 1) { sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); + } + + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); } pthread_mutex_unlock(&pNode->mutex); @@ -364,63 +373,54 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); syncBroadcastStatus(pNode); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); return 0; } int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); - if (pNode == NULL) return 0; + if (rid <= 0) return 0; - int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); + SSyncNode *pNode = syncAcquireNode(rid); + if (pNode == NULL) return 0; - taosReleaseRef(tsSyncRefId, rid); + int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); + syncReleaseNode(pNode); return code; } void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncPeer *pPeer = pNode->pMaster; if (pPeer && pNode->quorum > 1) { - char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0}; - - SSyncHead *pHead = (SSyncHead *)msg; - pHead->type = TAOS_SMSG_FORWARD_RSP; - pHead->len = sizeof(SFwdRsp); - - SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead)); - pFwdRsp->version = version; - pFwdRsp->code = code; + SFwdRsp rsp; + syncBuildSyncFwdRsp(&rsp, pNode->vgId, version, code); - int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); - int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, msgLen); - - if (retLen == msgLen) { - sTrace("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version); + if (taosWriteMsg(pPeer->peerFd, &rsp, sizeof(SFwdRsp)) == sizeof(SFwdRsp)) { + sTrace("%s, forward-rsp is sent, code:0x%x hver:%" PRIu64, pPeer->id, code, version); } else { - sDebug("%s, failed to send forward ack, restart", pPeer->id); + sDebug("%s, failed to send forward-rsp, restart", pPeer->id); syncRestartConnection(pPeer); } } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #if 0 void syncRecover(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; // to do: add a few lines to check if recover is OK // if take this node to unsync state, the whole system may not work nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); nodeVersion = 0; pthread_mutex_lock(&pNode->mutex); @@ -434,12 +434,12 @@ void syncRecover(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #endif int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return -1; pNodesRole->selfIndex = pNode->selfIndex; @@ -448,8 +448,7 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { pNodesRole->role[i] = pNode->peerInfo[i]->role; } - taosReleaseRef(tsSyncRefId, rid); - + syncReleaseNode(pNode); return 0; } @@ -479,11 +478,18 @@ static void syncAddArbitrator(SSyncNode *pNode) { } } - pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo); + pPeer = syncAddPeer(pNode, &nodeInfo); + if (pPeer != NULL) { + pPeer->isArb = 1; + sInfo("%s, is added as arbitrator", pPeer->id); + } + + pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = pPeer; } static void syncFreeNode(void *param) { SSyncNode *pNode = param; + sDebug("vgId:%d, node is freed, refCount:%d", pNode->vgId, pNode->refCount); pthread_mutex_destroy(&pNode->mutex); tfree(pNode->pRecv); @@ -491,28 +497,61 @@ static void syncFreeNode(void *param) { tfree(pNode); } -void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); } +SSyncNode *syncAcquireNode(int64_t rid) { + SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid); + if (pNode == NULL) { + sDebug("failed to acquire node from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, acquire node refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount); + } -int32_t syncDecPeerRef(SSyncPeer *pPeer) { - if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) { - taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); + return pNode; +} - sDebug("%s, resource is freed", pPeer->id); - tfree(pPeer); - return 0; +void syncReleaseNode(SSyncNode *pNode) { + int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, release node refId:%" PRId64 ", refCount:%d", pNode->vgId, pNode->rid, refCount); + + taosReleaseRef(tsNodeRefId, pNode->rid); +} + +static void syncFreePeer(void *param) { + SSyncPeer *pPeer = param; + sDebug("%s, peer is freed, refCount:%d", pPeer->id, pPeer->refCount); + + syncReleaseNode(pPeer->pSyncNode); + tfree(pPeer); +} + +SSyncPeer *syncAcquirePeer(int64_t rid) { + SSyncPeer *pPeer = taosAcquireRef(tsPeerRefId, rid); + if (pPeer == NULL) { + sDebug("failed to acquire peer from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pPeer->refCount, 1); + sTrace("%s, acquire peer refId:%" PRId64 ", refCount:%d", pPeer->id, rid, refCount); } - return 1; + return pPeer; +} + +void syncReleasePeer(SSyncPeer *pPeer) { + int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1); + sTrace("%s, release peer refId:%" PRId64 ", refCount:%d", pPeer->id, pPeer->rid, refCount); + + taosReleaseRef(tsPeerRefId, pPeer->rid); } static void syncClosePeerConn(SSyncPeer *pPeer) { sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd); taosTmrStopA(&pPeer->timer); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; - taosFreeTcpConn(pPeer->pConn); + void *pConn = pPeer->pConn; + if (pConn != NULL) syncFreeTcpConn(pPeer->pConn); } } @@ -521,11 +560,32 @@ static void syncRemovePeer(SSyncPeer *pPeer) { pPeer->ip = 0; syncClosePeerConn(pPeer); - syncDecPeerRef(pPeer); + //taosRemoveRef(tsPeerRefId, pPeer->rid); + syncReleasePeer(pPeer); +} + +static void syncStartCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; + + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); + if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { + int32_t checkMs = 100 + (pNode->vgId * 10) % 100; + + sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); + taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); + } +} + +static void syncStopCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + + taosTmrStopA(&pPeer->timer); + sDebug("%s, stop check peer connection", pPeer->id); } static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { - uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn); + uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ip == 0xFFFFFFFF) { sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno)); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -547,17 +607,11 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->role = TAOS_SYNC_ROLE_OFFLINE; pPeer->pSyncNode = pNode; pPeer->refCount = 1; + pPeer->rid = taosAddRef(tsPeerRefId, pPeer); - sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port); - int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { - int32_t checkMs = 100 + (pNode->vgId * 10) % 100; - if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; - sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); - taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer); - } + sInfo("%s, %p it is configured, ep:%s:%u rid:%" PRId64, pPeer->id, pPeer, pPeer->fqdn, pPeer->port, pPeer->rid); - taosAcquireRef(tsSyncRefId, pNode->rid); + (void)syncAcquireNode(pNode->rid); return pPeer; } @@ -574,8 +628,8 @@ static void syncResetFlowCtrl(SSyncNode *pNode) { pNode->peerInfo[index]->numOfRetrieves = 0; } - if (pNode->notifyFlowCtrl) { - (*pNode->notifyFlowCtrl)(pNode->vgId, 0); + if (pNode->notifyFlowCtrlFp) { + (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); } } @@ -599,13 +653,21 @@ static void syncChooseMaster(SSyncNode *pNode) { index = i; } } + sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index); + } else { + sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica); } // add arbitrator connection SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; - if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) { - onlineNum++; - replica = pNode->replica + 1; + if (pArb) { + if (pArb->role != TAOS_SYNC_ROLE_OFFLINE) { + onlineNum++; + replica = pNode->replica + 1; + sDebug("vgId:%d, arb:%s is used while choose master", pNode->vgId, pArb->id); + } else { + sError("vgId:%d, arb:%s is not used while choose master for its offline", pNode->vgId, pArb->id); + } } if (index < 0 && onlineNum > replica / 2.0) { @@ -619,6 +681,11 @@ static void syncChooseMaster(SSyncNode *pNode) { } } } + + if (index >= 0) { + sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId, + pNode->peerInfo[index]->id, index, onlineNum, replica); + } } if (index >= 0) { @@ -630,7 +697,7 @@ static void syncChooseMaster(SSyncNode *pNode) { taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER); syncResetFlowCtrl(pNode); - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } else { pPeer = pNode->peerInfo[index]; sInfo("%s, it shall work as master", pPeer->id); @@ -660,9 +727,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->vgId, nodeRole); - sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); + if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) { + sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } else { + nodeRole = TAOS_SYNC_ROLE_UNSYNCED; + sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } } else { for (int32_t index = 0; index < pNode->replica; ++index) { @@ -670,11 +741,14 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; if (masterIndex < 0) { masterIndex = index; + sDebug("vgId:%d, peer:%s is master, index:%d", pNode->vgId, pTemp->id, index); } else { // multiple masters, it shall not happen if (masterIndex == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); + } else { + sError("vgId:%d, peer:%s is master too, masterIndex:%d index:%d", pNode->vgId, pTemp->id, masterIndex, index); } } } @@ -691,7 +765,7 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); code = -1; for (int32_t index = 0; index < pNode->replica; ++index) { @@ -717,7 +791,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new if (pMaster) { // master is there pNode->pMaster = pMaster; - sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); + sDebug("%s, it is the master, replica:%d sver:%" PRIu64, pMaster->id, pNode->replica, pMaster->version); if (syncValidateMaster(pPeer) < 0) return; @@ -728,7 +802,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } else { sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); @@ -750,10 +824,10 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } if (consistent) { - sDebug("vgId:%d, choose master", pNode->vgId); + sDebug("vgId:%d, choose master, replica:%d", pNode->vgId, pNode->replica); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inequality, replica:%d", pNode->vgId, pNode->replica); } } @@ -762,7 +836,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } if (oldPeerRole != newPeerRole || nodeRole != oldSelfRole) { - sDebug("vgId:%d, roles changed, broadcast status", pNode->vgId); + sDebug("vgId:%d, roles changed, broadcast status, replica:%d", pNode->vgId, pNode->replica); syncBroadcastStatus(pNode); } @@ -782,26 +856,30 @@ static void syncRestartPeer(SSyncPeer *pPeer) { int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { sDebug("%s, check peer connection in 1000 ms", pPeer->id); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } void syncRestartConnection(SSyncPeer *pPeer) { if (pPeer->ip == 0) return; + if (syncAcquirePeer(pPeer->rid) == NULL) return; + syncRestartPeer(pPeer); syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE); + + syncReleasePeer(pPeer); } static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - sDebug("%s, sync-req is received", pPeer->id); + sInfo("%s, sync-req is received", pPeer->id); if (pPeer->ip == 0) return; if (nodeRole != TAOS_SYNC_ROLE_MASTER) { sError("%s, I am not master anymore", pPeer->id); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); return; } @@ -811,25 +889,30 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { } // start a new thread to retrieve the data - syncAddPeerRef(pPeer); + (void)syncAcquirePeer(pPeer->rid); + pthread_attr_t thattr; pthread_t thread; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); + int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); - if (ret != 0) { - sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno)); - syncDecPeerRef(pPeer); + if (ret < 0) { + sError("%s, failed to create sync retrieve thread since %s", pPeer->id, strerror(errno)); + syncReleasePeer(pPeer); } else { pPeer->sstatus = TAOS_SYNC_STATUS_START; - sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + sDebug("%s, sync retrieve thread:0x%08" PRIx64 " create successfully, rid:%" PRId64 ", set sstatus:%s", pPeer->id, + taosGetPthreadId(thread), pPeer->rid, syncStatus[pPeer->sstatus]); } } static void syncNotStarted(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -838,15 +921,22 @@ static void syncNotStarted(void *param, void *tmrId) { sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncTryRecoverFromMaster(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncRecoverFromMaster(SSyncPeer *pPeer) { @@ -860,34 +950,28 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); // Ensure the sync of mnode not interrupted - if (pNode->vgId != 1 && tsSyncNum >= tsMaxSyncNum) { + if (pNode->vgId != 1 && tsSyncNum >= SYNC_MAX_NUM) { sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum); - taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } sDebug("%s, try to sync", pPeer->id); - SFirstPkt firstPkt; - memset(&firstPkt, 0, sizeof(firstPkt)); - firstPkt.syncHead.type = TAOS_SMSG_SYNC_REQ; - firstPkt.syncHead.vgId = pNode->vgId; - firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead); - tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); - firstPkt.port = tsSyncPort; - taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + SSyncMsg msg; + syncBuildSyncReqMsg(&msg, pNode->vgId); + + taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - if (taosWriteMsg(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { + if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { sError("%s, failed to send sync-req to peer", pPeer->id); } else { - nodeSStatus = TAOS_SYNC_STATUS_START; - sInfo("%s, sync-req is sent to peer, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, sync-req is sent to peer, tranId:%u, sstatus:%s", pPeer->id, msg.tranId, syncStatus[nodeSStatus]); } } -static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { +static void syncProcessFwdResponse(SFwdRsp *pFwdRsp, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFwdRsp * pFwdRsp = (SFwdRsp *)cont; SSyncFwds *pSyncFwds = pNode->pSyncFwds; SFwdInfo * pFwdInfo; @@ -897,24 +981,25 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) { // find the forwardInfo from first for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { - pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % tsMaxFwdInfo; - if (pFwdRsp->version == pFwdInfo->version) break; + pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % SYNC_MAX_FWDS; + if (pFwdRsp->version == pFwdInfo->version) { + syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code); + syncRemoveConfirmedFwdInfo(pNode); + return; + } } - - syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code); - syncRemoveConfirmedFwdInfo(pNode); } } static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SWalHead * pHead = (SWalHead *)cont; + SWalHead * pHead = (SWalHead *)(cont + sizeof(SSyncHead)); sTrace("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; - (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); @@ -925,9 +1010,8 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { } } -static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { - SSyncNode * pNode = pPeer->pSyncNode; - SPeersStatus *pPeersStatus = (SPeersStatus *)cont; +static void syncProcessPeersStatusMsg(SPeersStatus *pPeersStatus, SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; sDebug("%s, status is received, self:%s:%s:%" PRIu64 ", peer:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role], @@ -941,23 +1025,22 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { } } -static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { +static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead) { if (pPeer->peerFd < 0) return -1; int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead)); if (hlen != sizeof(SSyncHead)) { - sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen); + sDebug("%s, failed to read msg since %s, hlen:%d", pPeer->id, tstrerror(errno), hlen); return -1; } - // head.len = htonl(head.len); - if (pHead->len < 0) { - sError("%s, invalid pkt length, hlen:%d", pPeer->id, pHead->len); + int32_t code = syncCheckHead(pHead); + if (code != 0) { + sError("%s, failed to check msg head since %s, type:%d", pPeer->id, tstrerror(code), pHead->type); return -1; } - assert(pHead->len <= TSDB_MAX_WAL_SIZE); - int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); + int32_t bytes = taosReadMsg(pPeer->peerFd, (char *)pHead + sizeof(SSyncHead), pHead->len); if (bytes != pHead->len) { sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len); return -1; @@ -966,67 +1049,68 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return 0; } -static int32_t syncProcessPeerMsg(void *param, void *buffer) { - SSyncPeer *pPeer = param; - SSyncHead head; - char * cont = buffer; +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return -1; + SSyncHead *pHead = buffer; SSyncNode *pNode = pPeer->pSyncNode; + pthread_mutex_lock(&pNode->mutex); - int32_t code = syncReadPeerMsg(pPeer, &head, cont); + int32_t code = syncReadPeerMsg(pPeer, pHead); if (code == 0) { - if (head.type == TAOS_SMSG_FORWARD) { - syncProcessForwardFromPeer(cont, pPeer); - } else if (head.type == TAOS_SMSG_FORWARD_RSP) { - syncProcessFwdResponse(cont, pPeer); - } else if (head.type == TAOS_SMSG_SYNC_REQ) { - syncProcessSyncRequest(cont, pPeer); - } else if (head.type == TAOS_SMSG_STATUS) { - syncProcessPeersStatusMsg(cont, pPeer); + if (pHead->type == TAOS_SMSG_SYNC_FWD) { + syncProcessForwardFromPeer(buffer, pPeer); + } else if (pHead->type == TAOS_SMSG_SYNC_FWD_RSP) { + syncProcessFwdResponse(buffer, pPeer); + } else if (pHead->type == TAOS_SMSG_SYNC_REQ) { + syncProcessSyncRequest(buffer, pPeer); + } else if (pHead->type == TAOS_SMSG_STATUS) { + syncProcessPeersStatusMsg(buffer, pPeer); } } pthread_mutex_unlock(&pNode->mutex); + syncReleasePeer(pPeer); return code; } -#define statusMsgLen sizeof(SSyncHead) + sizeof(SPeersStatus) + sizeof(SPeerStatus) * TAOS_SYNC_MAX_REPLICA - -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { - SSyncNode *pNode = pPeer->pSyncNode; - char msg[statusMsgLen] = {0}; - - if (pPeer->peerFd < 0 || pPeer->ip == 0) return; +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { + if (pPeer->peerFd < 0 || pPeer->ip == 0) { + sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd); + syncRestartConnection(pPeer); + return -1; + } - SSyncHead * pHead = (SSyncHead *)msg; - SPeersStatus *pPeersStatus = (SPeersStatus *)(msg + sizeof(SSyncHead)); + SSyncNode * pNode = pPeer->pSyncNode; + SPeersStatus msg; - pHead->type = TAOS_SMSG_STATUS; - pHead->len = statusMsgLen - sizeof(SSyncHead); + memset(&msg, 0, sizeof(SPeersStatus)); + syncBuildPeersStatus(&msg, pNode->vgId); - pPeersStatus->version = nodeVersion; - pPeersStatus->role = nodeRole; - pPeersStatus->ack = ack; - pPeersStatus->type = type; - pPeersStatus->tranId = tranId; + msg.role = nodeRole; + msg.ack = ack; + msg.type = type; + msg.tranId = tranId; + msg.version = nodeVersion; for (int32_t i = 0; i < pNode->replica; ++i) { - pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role; - pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version; + msg.peersStatus[i].role = pNode->peerInfo[i]->role; + msg.peersStatus[i].version = pNode->peerInfo[i]->version; } - int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); - if (retLen == statusMsgLen) { + if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SPeersStatus)) == sizeof(SPeersStatus)) { sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], - syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId, - statusType[pPeersStatus->type], pPeer->peerFd); + syncStatus[pPeer->sstatus], pPeer->version, ack, tranId, statusType[type], pPeer->peerFd); + return 0; } else { sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); + return -1; } } @@ -1040,36 +1124,34 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); - if (connFd < 0) { + SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + if (connFd <= 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } - SFirstPkt firstPkt; - memset(&firstPkt, 0, sizeof(firstPkt)); - firstPkt.syncHead.vgId = pPeer->nodeId ? pNode->vgId : 0; - firstPkt.syncHead.type = TAOS_SMSG_STATUS; - tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); - firstPkt.port = tsSyncPort; - firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId + SSyncMsg msg; + syncBuildSyncSetupMsg(&msg, pPeer->nodeId ? pNode->vgId : 0); - if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { - sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d", pPeer->id, connFd, pPeer->syncFd); + if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) == sizeof(SSyncMsg)) { + sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, msg.tranId); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; - pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); + if (pPeer->isArb) tsArbOnline = 1; } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); - taosClose(connFd); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosCloseSocket(connFd); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } static void syncCheckPeerConnection(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -1078,6 +1160,8 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { syncSetupPeerConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { @@ -1088,34 +1172,64 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - syncAddPeerRef(pPeer); - int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); + (void)syncAcquirePeer(pPeer->rid); + + int32_t ret = pthread_create(&thread, &thattr, (void *)syncRestoreData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); if (ret < 0) { - sError("%s, failed to create sync thread", pPeer->id); - taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); + SSyncNode *pNode = pPeer->pSyncNode; + nodeSStatus = TAOS_SYNC_STATUS_INIT; + sError("%s, failed to create sync restore thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + taosCloseSocket(pPeer->syncFd); + syncReleasePeer(pPeer); } else { - sInfo("%s, sync connection is up", pPeer->id); + sInfo("%s, sync restore thread:0x%08" PRIx64 " create successfully, rid:%" PRId64, pPeer->id, + taosGetPthreadId(thread), pPeer->rid); + } +} + +void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd) { + sInfo("recv sync test msg"); + + SSyncMsg rsp; + syncBuildSyncTestMsg(&rsp, -1); + if (taosWriteMsg(connFd, &rsp, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sInfo("failed to send sync test rsp since %s", strerror(errno)); } + + sInfo("send sync test rsp"); + taosMsleep(1000); + taosCloseSocket(connFd); } -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); - SFirstPkt firstPkt; - if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { - sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno)); + SSyncMsg msg; + if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sError("failed to read peer sync msg from ip:%s since %s", ipstr, strerror(errno)); + taosCloseSocket(connFd); + return; + } + + int32_t code = syncCheckHead((SSyncHead *)(&msg)); + if (code != 0) { + sError("failed to check peer sync msg from ip:%s since %s", ipstr, strerror(code)); taosCloseSocket(connFd); return; } - int32_t vgId = firstPkt.syncHead.vgId; + if (msg.head.type == TAOS_SMSG_TEST) { + syncProcessTestMsg(&msg, connFd); + return; + } + + int32_t vgId = msg.head.vgId; SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { sError("vgId:%d, vgId could not be found", vgId); @@ -1123,31 +1237,35 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } + sDebug("vgId:%d, sync connection is incoming, tranId:%u", vgId, msg.tranId); + SSyncNode *pNode = *ppNode; pthread_mutex_lock(&pNode->mutex); SSyncPeer *pPeer; for (i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; - if (pPeer && (strcmp(pPeer->fqdn, firstPkt.fqdn) == 0) && (pPeer->port == firstPkt.port)) break; + if (pPeer && (strcmp(pPeer->fqdn, msg.fqdn) == 0) && (pPeer->port == msg.port)) break; } pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL; if (pPeer == NULL) { - sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, firstPkt.fqdn, firstPkt.port); + sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, msg.fqdn, msg.port); taosCloseSocket(connFd); // syncSendVpeerCfgMsg(sync); } else { // first packet tells what kind of link - if (firstPkt.syncHead.type == TAOS_SMSG_SYNC_DATA) { + if (msg.head.type == TAOS_SMSG_SYNC_DATA) { pPeer->syncFd = connFd; + nodeSStatus = TAOS_SYNC_STATUS_START; + sInfo("%s, sync-data msg from master is received, tranId:%u, set sstatus:%s", pPeer->id, msg.tranId, + syncStatus[nodeSStatus]); syncCreateRestoreDataThread(pPeer); } else { sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; - pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); sDebug("%s, ready to exchange data", pPeer->id); syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId()); } @@ -1156,36 +1274,39 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pthread_mutex_unlock(&pNode->mutex); } -static void syncProcessBrokenLink(void *param) { - if (param == NULL) return; // the connection for arbitrator - SSyncPeer *pPeer = param; +static void syncProcessBrokenLink(int64_t rid) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&pNode->mutex); sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); pPeer->peerFd = -1; - - if (syncDecPeerRef(pPeer) != 0) { - syncRestartConnection(pPeer); + if (pPeer->isArb) { + tsArbOnline = 0; } + syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, pNode->rid); + + syncReleasePeer(pPeer); } -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { +static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; int64_t time = taosGetTimestampMs(); - if (pSyncFwds->fwds >= tsMaxFwdInfo) { - pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo; - pSyncFwds->fwds--; + if (pSyncFwds->fwds >= SYNC_MAX_FWDS) { + // pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; + // pSyncFwds->fwds--; + sError("vgId:%d, failed to save fwd info, hver:%" PRIu64 " fwds:%d", pNode->vgId, version, pSyncFwds->fwds); + return TSDB_CODE_SYN_TOO_MANY_FWDINFO; } if (pSyncFwds->fwds > 0) { - pSyncFwds->last = (pSyncFwds->last + 1) % tsMaxFwdInfo; + pSyncFwds->last = (pSyncFwds->last + 1) % SYNC_MAX_FWDS; } SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last; @@ -1196,6 +1317,8 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { pSyncFwds->fwds++; sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); + + return 0; } static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { @@ -1206,11 +1329,10 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; if (pFwdInfo->confirmed == 0) break; - pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo; + pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; pSyncFwds->fwds--; if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last; - // sDebug("vgId:%d, fwd info is removed, hver:%d, fwds:%d", - // pNode->vgId, pFwdInfo->version, pSyncFwds->fwds); + sTrace("vgId:%d, fwd info is removed, hver:%" PRIu64 " fwds:%d", pNode->vgId, pFwdInfo->version, pSyncFwds->fwds); memset(pFwdInfo, 0, sizeof(SFwdInfo)); } } @@ -1232,7 +1354,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } if (confirm && pFwdInfo->confirmed == 0) { - sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); + sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:0x%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); (*pNode->confirmForward)(pNode->vgId, pFwdInfo->mhandle, pFwdInfo->code); pFwdInfo->confirmed = 1; } @@ -1240,7 +1362,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code static void syncMonitorNodeRole(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; for (int32_t index = 0; index < pNode->replica; index++) { @@ -1257,12 +1379,12 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { } pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static void syncMonitorFwdInfos(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncFwds *pSyncFwds = pNode->pSyncFwds; @@ -1273,7 +1395,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { if (pSyncFwds->fwds > 0) { pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { - SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; + SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS; if (ABS(time - pFwdInfo->time) < 2000) break; sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, @@ -1288,7 +1410,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) { @@ -1308,24 +1430,24 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } } - return TSDB_CODE_SYN_INVALID_VERSION; + if (pNode->replica != 1) { + return TSDB_CODE_SYN_INVALID_VERSION; + } } // always update version - sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, + sTrace("vgId:%d, update version, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); nodeVersion = pWalHead->version; if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; - // only pkt from RPC or CQ can be forwarded + // only msg from RPC or CQ can be forwarded if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; // a hacker way to improve the performance pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); - pSyncHead->type = TAOS_SMSG_FORWARD; - pSyncHead->pversion = 0; - pSyncHead->len = sizeof(SWalHead) + pWalHead->len; + syncBuildSyncFwdMsg(pSyncHead, pNode->vgId, sizeof(SWalHead) + pWalHead->len); fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head pthread_mutex_lock(&pNode->mutex); @@ -1336,8 +1458,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; if (pNode->quorum > 1 && code == 0) { - syncSaveFwdInfo(pNode, pWalHead->version, mhandle); - code = 1; + code = syncSaveFwdInfo(pNode, pWalHead->version, mhandle); + if (code >= 0) code = 1; } int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); @@ -1355,4 +1477,3 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle return code; } - diff --git a/src/sync/src/syncMsg.c b/src/sync/src/syncMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..3348f1ec337e1b7c292d231360765c914936625c --- /dev/null +++ b/src/sync/src/syncMsg.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" +#include "tchecksum.h" +#include "syncInt.h" + +char *statusType[] = { + "broadcast", + "broadcast-rsp", + "setup-conn", + "setup-conn-rsp", + "exchange-data", + "exchange-data-rsp", + "check-role", + "check-role-rsp" +}; + +uint16_t syncGenTranId() { + return taosRand() & 0XFFFF; +} + +static void syncBuildHead(SSyncHead *pHead) { + pHead->protocol = SYNC_PROTOCOL_VERSION; + pHead->signature = SYNC_SIGNATURE; + pHead->code = 0; + pHead->cId = 0; + taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); +} + +int32_t syncCheckHead(SSyncHead *pHead) { + if (pHead->protocol != SYNC_PROTOCOL_VERSION) return TSDB_CODE_SYN_MISMATCHED_PROTOCOL; + if (pHead->signature != SYNC_SIGNATURE) return TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + if (pHead->cId != 0) return TSDB_CODE_SYN_MISMATCHED_CLUSTERID; + if (pHead->len <= 0 || pHead->len > TSDB_MAX_WAL_SIZE) return TSDB_CODE_SYN_INVALID_MSGLEN; + if (pHead->type <= TAOS_SMSG_START || pHead->type >= TAOS_SMSG_END) return TSDB_CODE_SYN_INVALID_MSGTYPE; + if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SSyncHead))) return TSDB_CODE_SYN_INVALID_CHECKSUM; + + return TSDB_CODE_SUCCESS; +} + +void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len) { + pHead->type = TAOS_SMSG_SYNC_FWD; + pHead->vgId = vgId; + pHead->len = len; + syncBuildHead(pHead); +} + +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code) { + pMsg->head.type = TAOS_SMSG_SYNC_FWD_RSP; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SFwdRsp) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); + + pMsg->version = version; + pMsg->code = code; +} + +static void syncBuildMsg(SSyncMsg *pMsg, int32_t vgId, ESyncMsgType type) { + pMsg->head.type = type; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SSyncMsg) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); + + pMsg->port = tsSyncPort; + pMsg->tranId = syncGenTranId(); + pMsg->sourceId = vgId; + tstrncpy(pMsg->fqdn, tsNodeFqdn, TSDB_FQDN_LEN); +} + +void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_REQ); } +void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_DATA); } +void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SETUP); } +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_TEST); } + +void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId) { + pMsg->head.type = TAOS_SMSG_STATUS; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SPeersStatus) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); +} + +void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) { + pMsg->head.type = TAOS_SMSG_SYNC_FILE_RSP; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SFileAck) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); +} + +void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId) { + pMsg->head.type = TAOS_SMSG_SYNC_FILE; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SFileVersion) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); +} \ No newline at end of file diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index d156c93865a3f4757cd9c9af0aff7b6789f59822..c0d66316cd5b802ddcaddf1015d4ceca1aa3b2c5 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -25,126 +25,47 @@ #include "tsync.h" #include "syncInt.h" -static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex) { - char name[TSDB_FILENAME_LEN * 2] = {0}; - char fname[TSDB_FILENAME_LEN * 3] = {0}; - uint32_t magic; - uint64_t fversion; - int64_t size; - uint32_t index = sindex; +static int32_t syncRecvFileVersion(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - if (sindex < 0 || eindex < sindex) return; - - while (1) { - name[0] = 0; - magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion); - if (magic == 0) break; - - snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name); - (void)remove(fname); - sDebug("%s, %s is removed", pPeer->id, fname); + SFileVersion fileVersion; + memset(&fileVersion, 0, sizeof(SFileVersion)); + int32_t ret = taosReadMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion)); + if (ret != sizeof(SFileVersion)) { + sError("%s, failed to read fver since %s", pPeer->id, strerror(errno)); + return -1; + } - index++; - if (index > eindex) break; + SFileAck fileVersionAck; + memset(&fileVersionAck, 0, sizeof(SFileAck)); + syncBuildFileAck(&fileVersionAck, pNode->vgId); + ret = taosWriteMsg(pPeer->syncFd, &fileVersionAck, sizeof(SFileAck)); + if (ret != sizeof(SFileAck)) { + sError("%s, failed to write fver ack since %s", pPeer->id, strerror(errno)); + return -1; } + + *fversion = htobe64(fileVersion.fversion); + return 0; } static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */ - SFileInfo sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */ - SFileAck fileAck = {0}; - int32_t code = -1; - char name[TSDB_FILENAME_LEN * 2] = {0}; - uint32_t pindex = 0; // index in last restore - bool fileChanged = false; - - *fversion = 0; - sinfo.index = 0; - while (1) { - // read file info - int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); - if (ret < 0) { - sError("%s, failed to read file info while restore file since %s", pPeer->id, strerror(errno)); - break; - } - - // if no more file from master, break; - if (minfo.name[0] == 0 || minfo.magic == 0) { - sDebug("%s, no more files to restore", pPeer->id); - - // remove extra files after the current index - syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX); - code = 0; - break; - } - - // remove extra files on slave between the current and last index - syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1); - pindex = minfo.index; - - // check the file info - sinfo = minfo; - sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size); - sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, - &sinfo.fversion); - - // if file not there or magic is not the same, file shall be synced - memset(&fileAck, 0, sizeof(fileAck)); - fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1 : 0; - - // send file ack - ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); - if (ret < 0) { - sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno)); - break; - } - - // if sync is not required, continue - if (fileAck.sync == 0) { - sDebug("%s, %s is the same", pPeer->id, minfo.name); - continue; - } - - // if sync is required, open file, receive from master, and write to file - // get the full path to file - minfo.name[sizeof(minfo.name) - 1] = 0; - snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name); - - int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); - if (dfd < 0) { - sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); - break; - } - ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size); - fsync(dfd); - close(dfd); - if (ret < 0) { - sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); - break; - } - - fileChanged = true; - sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); - } - - if (code == 0 && fileChanged) { - // data file is changed, code shall be set to 1 - *fversion = minfo.fversion; - code = 1; - sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion); + if (pNode->recvFileFp && (*pNode->recvFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) { + sError("%s, failed to restore file", pPeer->id); + return -1; } - if (code < 0) { - sError("%s, failed to restore %s since %s", pPeer->id, name, strerror(errno)); + if (syncRecvFileVersion(pPeer, fversion) < 0) { + return -1; } - return code; + sInfo("%s, all files are restored, fver:%" PRIu64, pPeer->id, *fversion); + return 0; } -static int32_t syncRestoreWal(SSyncPeer *pPeer) { +static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) { SSyncNode *pNode = pPeer->pSyncNode; int32_t ret, code = -1; uint64_t lastVer = 0; @@ -154,7 +75,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); - if (ret < 0) { + if (ret != sizeof(SWalHead)) { sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno)); break; } @@ -166,7 +87,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } // wal sync over ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); - if (ret < 0) { + if (ret != pHead->len) { sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno)); break; } @@ -179,7 +100,11 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } lastVer = pHead->version; - (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL); + ret = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL); + if (ret != 0) { + sError("%s, failed to restore record since %s, hver:%" PRIu64, pPeer->id, tstrerror(ret), pHead->version); + break; + } } if (code < 0) { @@ -187,6 +112,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } free(pHead); + *wver = lastVer; return code; } @@ -194,7 +120,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)offset; - (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); offset += pHead->len + sizeof(SWalHead); return offset; @@ -205,6 +131,11 @@ static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) { SRecvBuffer *pRecv = pNode->pRecv; int32_t forwards = 0; + if (pRecv == NULL) { + sError("%s, recv buffer is null, restart connect", pPeer->id); + return -1; + } + sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards); char *offset = pRecv->buffer; @@ -253,6 +184,7 @@ int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { + sDebug("vgId:%d, recv buffer:%p is freed", pNode->vgId, pNode->pRecv); tfree(pNode->pRecv->buffer); } @@ -277,6 +209,7 @@ static int32_t syncOpenRecvBuffer(SSyncNode *pNode) { pNode->pRecv = pRecv; + sDebug("vgId:%d, recv buffer:%p is created", pNode->vgId, pNode->pRecv); return 0; } @@ -286,35 +219,39 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { uint64_t fversion = 0; sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - SFirstPktRsp firstPktRsp = {.sync = 1}; - if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { - sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + SSyncRsp rsp = {.sync = 1, .tranId = syncGenTranId()}; + if (taosWriteMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) { + sError("%s, failed to send sync rsp since %s", pPeer->id, strerror(errno)); return -1; } + sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId); sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + (*pNode->startSyncFileFp)(pNode->vgId); + int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { - sError("%s, failed to restore file", pPeer->id); + (*pNode->stopSyncFileFp)(pNode->vgId, fversion); + sError("%s, failed to restore files", pPeer->id); return -1; } - // if code > 0, data file is changed, notify app, and pass the version - if (code > 0 && pNode->notifyFileSynced) { - if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) { - sError("%s, app not in ready state", pPeer->id); - return -1; - } - } - + (*pNode->stopSyncFileFp)(pNode->vgId, fversion); nodeVersion = fversion; - sInfo("%s, start to restore wal", pPeer->id); - if (syncRestoreWal(pPeer) < 0) { - sError("%s, failed to restore wal", pPeer->id); + sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); + uint64_t wver = 0; + code = syncRestoreWal(pPeer, &wver); // lastwar + if (code < 0) { + sError("%s, failed to restore wal, code:%d", pPeer->id, code); return -1; } + if (wver != 0) { + nodeVersion = wver; + sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion); + } + nodeSStatus = TAOS_SYNC_STATUS_CACHE; sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { @@ -326,13 +263,21 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } void *syncRestoreData(void *param) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) { + sError("failed to restore data, invalid peer rid:%" PRId64, rid); + return NULL; + } + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); - __sync_fetch_and_add(&tsSyncNum, 1); + atomic_add_fetch_32(&tsSyncNum, 1); + sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); + nodeRole = TAOS_SYNC_ROLE_SYNCING; + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); if (syncOpenRecvBuffer(pNode) < 0) { sError("%s, failed to allocate recv buffer, restart connection", pPeer->id); @@ -349,15 +294,18 @@ void *syncRestoreData(void *param) { } } - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; - sInfo("%s, sync over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); syncCloseRecvBuffer(pNode); - __sync_fetch_and_sub(&tsSyncNum, 1); - syncDecPeerRef(pPeer); + atomic_sub_fetch_32(&tsSyncNum, 1); + + // The ref is obtained in both the create thread and the current thread, so it is released twice + syncReleasePeer(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 36b197dd46484c7ce676391aea5dc4efaf00b9d8..be4073760dfa551878301bc3bd4f2405b12b77cf 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -14,7 +14,6 @@ */ #define _DEFAULT_SOURCE -#include #include "os.h" #include "taoserror.h" #include "tlog.h" @@ -28,7 +27,7 @@ static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); return -1; @@ -40,7 +39,7 @@ static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) { static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); return true; @@ -56,9 +55,9 @@ static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) { static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { - sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); + sDebug("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); return -1; } @@ -68,7 +67,7 @@ static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) { static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); pPeer->fileChanged = 1; @@ -85,91 +84,54 @@ static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { return false; } -static int32_t syncRetrieveFile(SSyncPeer *pPeer) { +static int32_t syncSendFileVersion(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo)); - SFileAck fileAck = {0}; - int32_t code = -1; - char name[TSDB_FILENAME_LEN * 2] = {0}; - - if (syncGetFileVersion(pNode, pPeer) < 0) return -1; - - while (1) { - // retrieve file info - fileInfo.name[0] = 0; - fileInfo.size = 0; - fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, - &fileInfo.size, &fileInfo.fversion); - // fileInfo.size = htonl(size); - sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); - - // send the file info - int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); - if (ret < 0) { - code = -1; - sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } - // if no file anymore, break - if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { - code = 0; - sDebug("%s, no more files to sync", pPeer->id); - break; - } - - // wait for the ack from peer - ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); - if (ret < 0) { - code = -1; - sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } + SFileVersion fileVersion; + memset(&fileVersion, 0, sizeof(SFileVersion)); + syncBuildFileVersion(&fileVersion, pNode->vgId); - // set the peer sync version - pPeer->sversion = fileInfo.fversion; + uint64_t fver = pPeer->lastFileVer; + fileVersion.fversion = htobe64(fver); + int32_t ret = taosWriteMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion)); + if (ret != sizeof(SFileVersion)) { + sError("%s, failed to write fver:%" PRIu64 " since %s", pPeer->id, fver, strerror(errno)); + return -1; + } - // if sync is not required, continue - if (fileAck.sync == 0) { - fileInfo.index++; - sDebug("%s, %s is the same", pPeer->id, fileInfo.name); - continue; - } + SFileAck fileAck; + memset(&fileAck, 0, sizeof(SFileAck)); + ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck)); + if (ret != sizeof(SFileAck)) { + sError("%s, failed to read fver ack since %s", pPeer->id, strerror(errno)); + return -1; + } - // get the full path to file - snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); + // set the peer sync version + pPeer->sversion = fver; - // send the file to peer - int32_t sfd = open(name, O_RDONLY); - if (sfd < 0) { - code = -1; - sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } + return 0; +} - ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); - close(sfd); - if (ret < 0) { - code = -1; - sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } +static int32_t syncRetrieveFile(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; - sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); - fileInfo.index++; + if (syncGetFileVersion(pNode, pPeer) < 0) { + pPeer->fileChanged = 1; + return -1; + } - // check if processed files are modified - if (syncAreFilesModified(pNode, pPeer)) { - code = -1; - break; - } + if (pNode->sendFileFp && (*pNode->sendFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) { + sError("%s, failed to retrieve file", pPeer->id); + return -1; } - if (code != TSDB_CODE_SUCCESS) { - sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code); + if (syncSendFileVersion(pPeer) < 0) { + return -1; } - return code; + sInfo("%s, all files are retrieved", pPeer->id); + return 0; } // if only a partial record is read out, upper layer will reload the file to get a complete record @@ -181,7 +143,7 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { } if (ret == 0) { - sTrace("sfd:%d, read to the end of file, ret:%d", sfd, ret); + sDebug("sfd:%d, read to the end of file, ret:%d", sfd, ret); return 0; } @@ -208,14 +170,14 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { return sizeof(SWalHead) + pHead->len; } -static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { - int32_t sfd = open(name, O_RDONLY); +static int64_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { + int32_t sfd = open(name, O_RDONLY | O_BINARY); if (sfd < 0) { sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno)); return -1; } - int32_t code = taosLSeek(sfd, offset, SEEK_SET); + int64_t code = taosLSeek(sfd, offset, SEEK_SET); if (code < 0) { sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno)); close(sfd); @@ -225,7 +187,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi sDebug("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion); SWalHead *pHead = malloc(SYNC_MAX_SIZE); - int32_t bytes = 0; + int64_t bytes = 0; while (1) { code = syncReadOneWalRecord(sfd, pHead); @@ -236,13 +198,13 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi if (code == 0) { code = bytes; - sDebug("%s, read to the end of wal, bytes:%d", pPeer->id, bytes); + sDebug("%s, read to the end of wal, bytes:%" PRId64, pPeer->id, bytes); break; } sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); - int32_t wsize = code; + int32_t wsize = (int32_t)code; int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); if (ret != wsize) { code = -1; @@ -266,7 +228,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi return code; } -static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { +static int64_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; int32_t once = 0; // last WAL has once ever been processed int64_t offset = 0; @@ -281,9 +243,9 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) if (syncAreFilesModified(pNode, pPeer)) return -1; if (syncGetWalVersion(pNode, pPeer) < 0) return -1; - int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset); + int64_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset); if (bytes < 0) { - sDebug("%s, failed to retrieve last wal", pPeer->id); + sDebug("%s, failed to retrieve last wal, bytes:%" PRId64, pPeer->id, bytes); return bytes; } @@ -301,7 +263,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) // if all data up to fversion is read out, it is over if (pPeer->sversion >= fversion && fversion > 0) { - sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d sver:%" PRIu64, pPeer->id, fversion, bytes, + sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%" PRId64 " sver:%" PRIu64, pPeer->id, fversion, bytes, pPeer->sversion); return 0; } @@ -309,32 +271,33 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) // if all data are read out, and no update if (bytes == 0 && !walModified) { // wal not closed, it means some data not flushed to disk, wait for a while - usleep(10000); + taosMsleep(10); } // if bytes > 0, file is updated, or fversion is not reached but file still open, read again once = 1; offset += bytes; - sDebug("%s, continue retrieve last wal, bytes:%d offset:%" PRId64, pPeer->id, bytes, offset); + sDebug("%s, continue retrieve last wal, bytes:%" PRId64 " offset:%" PRId64 " sver:%" PRIu64 " fver:%" PRIu64, pPeer->id, + bytes, offset, pPeer->sversion, fversion); } return -1; } -static int32_t syncRetrieveWal(SSyncPeer *pPeer) { +static int64_t syncRetrieveWal(SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; - int32_t code = -1; + int64_t code = -1; int64_t index = 0; while (1) { // retrieve wal info wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); + code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index); if (code < 0) { - sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code); + sError("%s, failed to get wal info since:%s, code:0x%" PRIx64, pPeer->id, strerror(errno), code); break; } @@ -346,6 +309,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { if (code == 0) { // last wal code = syncProcessLastWal(pPeer, wname, index); + sInfo("%s, last wal processed, code:%" PRId64, pPeer->id, code); break; } @@ -356,24 +320,24 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { struct stat fstat; if (stat(fname, &fstat) < 0) { code = -1; - sDebug("%s, failed to stat wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + sDebug("%s, failed to stat wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); break; } size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); - int32_t sfd = open(fname, O_RDONLY); + int32_t sfd = open(fname, O_RDONLY | O_BINARY); if (sfd < 0) { code = -1; - sError("%s, failed to open wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + sError("%s, failed to open wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); break; } - code = taosSendFile(pPeer->syncFd, sfd, NULL, size); + code = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, size); close(sfd); if (code < 0) { - sError("%s, failed to send wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + sError("%s, failed to send wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); break; } @@ -384,14 +348,17 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { } if (code == 0) { - pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; - sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - SWalHead walHead; memset(&walHead, 0, sizeof(walHead)); - taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); + if (taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)) == sizeof(walHead)) { + pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; + sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + } else { + sError("%s, failed to send last wal record since %s", pPeer->id, strerror(errno)); + code = -1; + } } else { - sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code); + sError("%s, failed to send wal since %s, code:0x%" PRIx64, pPeer->id, strerror(errno), code); } return code; @@ -400,24 +367,22 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFirstPkt firstPkt; - memset(&firstPkt, 0, sizeof(firstPkt)); - firstPkt.syncHead.type = TAOS_SMSG_SYNC_DATA; - firstPkt.syncHead.vgId = pNode->vgId; - tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); - firstPkt.port = tsSyncPort; + SSyncMsg msg; + syncBuildSyncDataMsg(&msg, pNode->vgId); - if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) < 0) { - sError("%s, failed to send sync firstPkt since %s", pPeer->id, strerror(errno)); + if (taosWriteMsg(pPeer->syncFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sError("%s, failed to send sync-data msg since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId); return -1; } + sDebug("%s, send sync-data msg to peer, tranId:%u", pPeer->id, msg.tranId); - SFirstPktRsp firstPktRsp; - if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { - sError("%s, failed to read sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + SSyncRsp rsp; + if (taosReadMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) { + sError("%s, failed to read sync-data rsp since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId); return -1; } + sDebug("%s, recv sync-data rsp from peer, tranId:%u rsp-tranId:%u", pPeer->id, msg.tranId, rsp.tranId); return 0; } @@ -440,9 +405,9 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { if (pPeer->sversion == 0) pPeer->sversion = 1; sInfo("%s, start to retrieve wals", pPeer->id); - int32_t code = syncRetrieveWal(pPeer); - if (code != 0) { - sError("%s, failed to retrieve wals, code:0x%x", pPeer->id, code); + int64_t code = syncRetrieveWal(pPeer); + if (code < 0) { + sError("%s, failed to retrieve wals, code:0x%" PRIx64, pPeer->id, code); return -1; } @@ -450,11 +415,20 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { } void *syncRetrieveData(void *param) { - SSyncPeer *pPeer = (SSyncPeer *)param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) { + sError("failed to retrieve data, invalid peer rid:%" PRId64, rid); + return NULL; + } + SSyncNode *pNode = pPeer->pSyncNode; + taosBlockSIGPIPE(); + sInfo("%s, start to retrieve data, sstatus:%s, numOfRetrieves:%d", pPeer->id, syncStatus[pPeer->sstatus], + pPeer->numOfRetrieves); - if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); + if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves); pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (pPeer->syncFd < 0) { @@ -474,12 +448,19 @@ void *syncRetrieveData(void *param) { pPeer->numOfRetrieves++; } else { pPeer->numOfRetrieves = 0; - if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0); + // if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); } + if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); + pPeer->fileChanged = 0; - taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); + taosCloseSocket(pPeer->syncFd); + + // The ref is obtained in both the create thread and the current thread, so it is released twice + sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + + syncReleasePeer(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/syncTcp.c similarity index 80% rename from src/sync/src/taosTcpPool.c rename to src/sync/src/syncTcp.c index eb05cf7c6f8d8d9a068b60c54f0ae1426e62c429..3ad9e9bba01eb861c48fa121b154af3e7838dda0 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/syncTcp.c @@ -19,15 +19,15 @@ #include "tutil.h" #include "tsocket.h" #include "taoserror.h" -#include "taosTcpPool.h" #include "twal.h" #include "tsync.h" #include "syncInt.h" +#include "syncTcp.h" typedef struct SThreadObj { pthread_t thread; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -37,22 +37,23 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - int32_t acceptFd; // FD for accept new connection + SOCKET acceptFd; // FD for accept new connection + int8_t stop; } SPoolObj; typedef struct { SThreadObj *pThread; - void * ahandle; - int32_t fd; + int64_t handleId; + SOCKET fd; int32_t closedByApp; } SConnObj; -static void *taosAcceptPeerTcpConnection(void *argv); -static void *taosProcessTcpData(void *param); -static void taosStopPoolThread(SThreadObj *pThread); -static SThreadObj *taosGetTcpThread(SPoolObj *pPool); +static void *syncAcceptPeerTcpConnection(void *argv); +static void *syncProcessTcpData(void *param); +static void syncStopPoolThread(SThreadObj *pThread); +static SThreadObj *syncGetTcpThread(SPoolObj *pPool); -void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { +void *syncOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_t thattr; SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); @@ -80,9 +81,9 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) { + if (pthread_create(&(pPool->thread), &thattr, (void *)syncAcceptPeerTcpConnection, pPool) != 0) { sError("failed to create accept thread for TCP server since %s", strerror(errno)); - close(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); tfree(pPool->pThread); tfree(pPool); return NULL; @@ -94,16 +95,28 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { return pPool; } -void taosCloseTcpThreadPool(void *param) { +void syncCloseTcpThreadPool(void *param) { SPoolObj * pPool = param; SThreadObj *pThread; + pPool->stop = 1; + +#ifdef WINDOWS + closesocket(pPool->acceptFd); +#elif defined(__APPLE__) + if (pPool->acceptFd!=-1) { + close(pPool->acceptFd); + pPool->acceptFd = -1; + } +#else shutdown(pPool->acceptFd, SHUT_RD); +#endif + pthread_join(pPool->thread, NULL); for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { pThread = pPool->pThread[i]; - if (pThread) taosStopPoolThread(pThread); + if (pThread) syncStopPoolThread(pThread); } sDebug("%p TCP pool is closed", pPool); @@ -112,7 +125,7 @@ void taosCloseTcpThreadPool(void *param) { tfree(pPool); } -void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -122,7 +135,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { return NULL; } - SThreadObj *pThread = taosGetTcpThread(pPool); + SThreadObj *pThread = syncGetTcpThread(pPool); if (pThread == NULL) { tfree(pConn); return NULL; @@ -130,7 +143,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { pConn->fd = connFd; pConn->pThread = pThread; - pConn->ahandle = pPeer; + pConn->handleId = rid; pConn->closedByApp = 0; event.events = EPOLLIN | EPOLLRDHUP; @@ -149,7 +162,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { return pConn; } -void taosFreeTcpConn(void *param) { +void syncFreeTcpConn(void *param) { SConnObj * pConn = param; SThreadObj *pThread = pConn->pThread; @@ -164,18 +177,18 @@ static void taosProcessBrokenLink(SConnObj *pConn) { SPoolInfo * pInfo = &pPool->info; if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR); - (*pInfo->processBrokenLink)(pConn->ahandle); + (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); - taosClose(pConn->fd); + taosCloseSocket(pConn->fd); tfree(pConn); } #define maxEvents 10 -static void *taosProcessTcpData(void *param) { +static void *syncProcessTcpData(void *param) { SThreadObj *pThread = (SThreadObj *)param; SPoolObj * pPool = pThread->pPool; SPoolInfo * pInfo = &pPool->info; @@ -221,8 +234,8 @@ static void *taosProcessTcpData(void *param) { } if (pConn->closedByApp == 0) { - if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) { - taosFreeTcpConn(pConn); + if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) { + syncFreeTcpConn(pConn); continue; } } @@ -233,13 +246,13 @@ static void *taosProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); - close(pThread->pollFd); + EpollClose(pThread->pollFd); tfree(pThread); tfree(buffer); return NULL; } -static void *taosAcceptPeerTcpConnection(void *argv) { +static void *syncAcceptPeerTcpConnection(void *argv) { SPoolObj * pPool = (SPoolObj *)argv; SPoolInfo *pInfo = &pPool->info; @@ -248,7 +261,12 @@ static void *taosAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + if (pPool->stop) { + sDebug("%p TCP server accept is stopped", pPool); + break; + } + if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); @@ -264,11 +282,11 @@ static void *taosAcceptPeerTcpConnection(void *argv) { (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } - taosClose(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); return NULL; } -static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { +static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { SThreadObj *pThread = pPool->pThread[pPool->nextId]; if (pThread) return pThread; @@ -277,7 +295,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = epoll_create(10); // size does not matter + pThread->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; @@ -286,11 +304,11 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread); + int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)syncProcessTcpData, pThread); pthread_attr_destroy(&thattr); if (ret != 0) { - close(pThread->pollFd); + EpollClose(pThread->pollFd); tfree(pThread); return NULL; } @@ -303,7 +321,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { return pThread; } -static void taosStopPoolThread(SThreadObj *pThread) { +static void syncStopPoolThread(SThreadObj *pThread) { pthread_t thread = pThread->thread; if (!taosCheckPthreadValid(thread)) { return; diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt index 27614454f9f8924de561f0907f06078b03708363..f2b05ab2263c0d80bc870981f86933922de639e4 100644 --- a/src/sync/test/CMakeLists.txt +++ b/src/sync/test/CMakeLists.txt @@ -13,4 +13,3 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(syncServer sync trpc common) ENDIF () - diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 5a64a0a36a9c5e0557e78216ba41a48a23cc874b..eeaa6a08c2e47d103b62d6023dde74341585f65f 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -100,7 +100,7 @@ int processRpcMsg(void *item) { pHead->msgType = pMsg->msgType; pHead->len = pMsg->contLen; - uDebug("ver:%" PRIu64 ", pkt from client processed", pHead->version); + uDebug("ver:%" PRIu64 ", rsp from client processed", pHead->version); writeIntoWal(pHead); syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC); @@ -275,7 +275,7 @@ int getWalInfo(int32_t vgId, char *name, int64_t *index) { int writeToCache(int32_t vgId, void *data, int type) { SWalHead *pHead = data; - uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type); + uDebug("rsp from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type); int msgSize = pHead->len + sizeof(SWalHead); void *pMsg = taosAllocateQitem(msgSize); @@ -296,11 +296,10 @@ void initSync() { pCfg->replica = 1; pCfg->quorum = 1; syncInfo.vgId = 1; - syncInfo.getFileInfo = getFileInfo; - syncInfo.getWalInfo = getWalInfo; - syncInfo.writeToCache = writeToCache; + syncInfo.getWalInfoFp = getWalInfo; + syncInfo.writeToCacheFp = writeToCache; syncInfo.confirmForward = confirmForward; - syncInfo.notifyRole = notifyRole; + syncInfo.notifyRoleFp = notifyRole; pCfg->nodeInfo[0].nodeId = 1; pCfg->nodeInfo[0].nodePort = 7010; diff --git a/src/tfs/CMakeLists.txt b/src/tfs/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b435c84366fb47bd137b1c13bc98eab625bbcc66 --- /dev/null +++ b/src/tfs/CMakeLists.txt @@ -0,0 +1,12 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(inc) +AUX_SOURCE_DIRECTORY(src SRC) +ADD_LIBRARY(tfs ${SRC}) +TARGET_LINK_LIBRARIES(tfs tutil) + +IF (TD_LINUX) + # Someone has no gtest directory, so comment it + # ADD_SUBDIRECTORY(tests) +ENDIF () diff --git a/src/tfs/inc/tfsint.h b/src/tfs/inc/tfsint.h new file mode 100644 index 0000000000000000000000000000000000000000..fa4cd597237a957064e1cca74fe089051db450c6 --- /dev/null +++ b/src/tfs/inc/tfsint.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TD_TFSINT_H +#define TD_TFSINT_H + +#include "tlog.h" +#include "tglobal.h" +#include "tfs.h" +#include "tcoding.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int fsDebugFlag; + +// For debug purpose +#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }} +#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }} +#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }} +#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", 255, __VA_ARGS__); }} +#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} +#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} + +// Global Definitions +#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 + +// tdisk.c ====================================================== +typedef struct { + int64_t size; + int64_t free; +} SDiskMeta; + +typedef struct SDisk { + int level; + int id; + char dir[TSDB_FILENAME_LEN]; + SDiskMeta dmeta; +} SDisk; + +#define DISK_LEVEL(pd) ((pd)->level) +#define DISK_ID(pd) ((pd)->id) +#define DISK_DIR(pd) ((pd)->dir) +#define DISK_META(pd) ((pd)->dmeta) +#define DISK_SIZE(pd) ((pd)->dmeta.size) +#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free) + +SDisk *tfsNewDisk(int level, int id, const char *dir); +SDisk *tfsFreeDisk(SDisk *pDisk); +int tfsUpdateDiskInfo(SDisk *pDisk); + +// ttier.c ====================================================== +typedef struct { + int64_t size; + int64_t free; + int16_t nAvailDisks; // # of Available disks +} STierMeta; +typedef struct STier { + pthread_spinlock_t lock; + int level; + int16_t ndisk; // # of disks mounted to this tier + int16_t nextid; // next disk id to allocate + STierMeta tmeta; + SDisk * disks[TSDB_MAX_DISKS_PER_TIER]; +} STier; + +#define TIER_LEVEL(pt) ((pt)->level) +#define TIER_NDISKS(pt) ((pt)->ndisk) +#define TIER_SIZE(pt) ((pt)->tmeta.size) +#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free) +#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks) +#define DISK_AT_TIER(pt, id) ((pt)->disks[id]) + +int tfsInitTier(STier *pTier, int level); +void tfsDestroyTier(STier *pTier); +SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); +void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta); +int tfsAllocDiskOnTier(STier *pTier); +void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta); +void tfsPosNextId(STier *pTier); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/tfs/src/tdisk.c b/src/tfs/src/tdisk.c new file mode 100644 index 0000000000000000000000000000000000000000..37798d3a886b443a20703747ad9a99e26c1502b7 --- /dev/null +++ b/src/tfs/src/tdisk.c @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" + +#include "taoserror.h" +#include "tfsint.h" + +// PROTECTED ==================================== +SDisk *tfsNewDisk(int level, int id, const char *dir) { + SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk)); + if (pDisk == NULL) { + terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + return NULL; + } + + pDisk->level = level; + pDisk->id = id; + strncpy(pDisk->dir, dir, TSDB_FILENAME_LEN); + + return pDisk; +} + +SDisk *tfsFreeDisk(SDisk *pDisk) { + if (pDisk) { + free(pDisk); + } + return NULL; +} + +int tfsUpdateDiskInfo(SDisk *pDisk) { + ASSERT(pDisk != NULL); + + SysDiskSize diskSize = {0}; + + int code = taosGetDiskSize(pDisk->dir, &diskSize); + if (code != 0) { + fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + } + + pDisk->dmeta.size = diskSize.tsize; + pDisk->dmeta.free = diskSize.avail; + + return code; +} diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c new file mode 100644 index 0000000000000000000000000000000000000000..7b7c9b6127db0f1c8cabab35254ba4a128e05f05 --- /dev/null +++ b/src/tfs/src/tfs.c @@ -0,0 +1,600 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "hash.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tfs.h" +#include "tfsint.h" + +#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) + +typedef struct { + pthread_spinlock_t lock; + SFSMeta meta; + int nlevel; + STier tiers[TSDB_MAX_TIERS]; + SHashObj * map; // name to did map +} SFS; + +typedef struct { + SDisk *pDisk; +} SDiskIter; + +#define TFS_META() (pfs->meta) +#define TFS_NLEVEL() (pfs->nlevel) +#define TFS_TIERS() (pfs->tiers) +#define TFS_TIER_AT(level) (TFS_TIERS() + (level)) +#define TFS_DISK_AT(level, id) DISK_AT_TIER(TFS_TIER_AT(level), id) +#define TFS_PRIMARY_DISK() TFS_DISK_AT(TFS_PRIMARY_LEVEL, TFS_PRIMARY_ID) +#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL())) +#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level)))) +#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id)) + +#define tfsLock() pthread_spin_lock(&(pfs->lock)) +#define tfsUnLock() pthread_spin_unlock(&(pfs->lock)) + +static SFS tfs = {0}; +static SFS *pfs = &tfs; + +// STATIC DECLARATION +static int tfsMount(SDiskCfg *pCfg); +static int tfsCheck(); +static int tfsCheckAndFormatCfg(SDiskCfg *pCfg); +static int tfsFormatDir(char *idir, char *odir); +static SDisk *tfsGetDiskByID(SDiskID did); +static SDisk *tfsGetDiskByName(const char *dir); +static int tfsOpendirImpl(TDIR *tdir); +static void tfsInitDiskIter(SDiskIter *pIter); +static SDisk *tfsNextDisk(SDiskIter *pIter); + +// FS APIs ==================================== +int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { + ASSERT(ndisk > 0); + + for (int level = 0; level < TSDB_MAX_TIERS; level++) { + if (tfsInitTier(TFS_TIER_AT(level), level) < 0) { + while (true) { + level--; + if (level < 0) break; + + tfsDestroyTier(TFS_TIER_AT(level)); + } + + return -1; + } + } + + pthread_spin_init(&(pfs->lock), 0); + + pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pfs->map == NULL) { + terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + tfsDestroy(); + return -1; + } + + for (int idisk = 0; idisk < ndisk; idisk++) { + if (tfsMount(pDiskCfg + idisk) < 0) { + tfsDestroy(); + return -1; + } + } + + if (tfsCheck() < 0) { + tfsDestroy(); + return -1; + } + + tfsUpdateInfo(NULL); + for (int level = 0; level < TFS_NLEVEL(); level++) { + tfsPosNextId(TFS_TIER_AT(level)); + } + + return 0; +} + +void tfsDestroy() { + taosHashCleanup(pfs->map); + pfs->map = NULL; + + pthread_spin_destroy(&(pfs->lock)); + for (int level = 0; level < TFS_NLEVEL(); level++) { + tfsDestroyTier(TFS_TIER_AT(level)); + } +} + +void tfsUpdateInfo(SFSMeta *pFSMeta) { + SFSMeta fsMeta; + STierMeta tierMeta; + + if (pFSMeta == NULL) { + pFSMeta = &fsMeta; + } + + memset(pFSMeta, 0, sizeof(*pFSMeta)); + + for (int level = 0; level < TFS_NLEVEL(); level++) { + STier *pTier = TFS_TIER_AT(level); + tfsUpdateTierInfo(pTier, &tierMeta); + pFSMeta->tsize += tierMeta.size; + pFSMeta->avail += tierMeta.free; + } + + tfsLock(); + pfs->meta = *pFSMeta; + tfsUnLock(); +} + +void tfsGetMeta(SFSMeta *pMeta) { + ASSERT(pMeta); + + tfsLock(); + *pMeta = pfs->meta; + tfsUnLock(); +} + +/* Allocate an existing available tier level + */ +void tfsAllocDisk(int expLevel, int *level, int *id) { + ASSERT(expLevel >= 0); + + *level = expLevel; + *id = TFS_UNDECIDED_ID; + + if (*level >= TFS_NLEVEL()) { + *level = TFS_NLEVEL() - 1; + } + + while (*level >= 0) { + *id = tfsAllocDiskOnTier(TFS_TIER_AT(*level)); + if (*id == TFS_UNDECIDED_ID) { + (*level)--; + continue; + } + + return; + } + + *level = TFS_UNDECIDED_LEVEL; + *id = TFS_UNDECIDED_ID; +} + +const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); } +const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); } + +// TFILE APIs ==================================== +void tfsInitFile(TFILE *pf, int level, int id, const char *bname) { + ASSERT(TFS_IS_VALID_DISK(level, id)); + + SDisk *pDisk = TFS_DISK_AT(level, id); + + pf->level = level; + pf->id = id; + strncpy(pf->rname, bname, TSDB_FILENAME_LEN); + + char tmpName[TMPNAME_LEN] = {0}; + snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname); + tstrncpy(pf->aname, tmpName, TSDB_FILENAME_LEN); +} + +bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) { + ASSERT(pf1 != NULL || pf2 != NULL); + if (pf1 == NULL || pf2 == NULL) return false; + if (pf1->level != pf2->level) return false; + if (pf1->id != pf2->id) return false; + if (strncmp(pf1->rname, pf2->rname, TSDB_FILENAME_LEN) != 0) return false; + return true; +} + +int tfsEncodeFile(void **buf, TFILE *pf) { + int tlen = 0; + + tlen += taosEncodeVariantI32(buf, pf->level); + tlen += taosEncodeVariantI32(buf, pf->id); + tlen += taosEncodeString(buf, pf->rname); + + return tlen; +} + +void *tfsDecodeFile(void *buf, TFILE *pf) { + int32_t level, id; + char * rname; + + buf = taosDecodeVariantI32(buf, &(level)); + buf = taosDecodeVariantI32(buf, &(id)); + buf = taosDecodeString(buf, &rname); + + tfsInitFile(pf, level, id, rname); + + tfree(rname); + return buf; +} + +void tfsbasename(const TFILE *pf, char *dest) { + char tname[TSDB_FILENAME_LEN] = "\0"; + + strncpy(tname, pf->aname, TSDB_FILENAME_LEN); + strncpy(dest, basename(tname), TSDB_FILENAME_LEN); +} + +void tfsdirname(const TFILE *pf, char *dest) { + char tname[TSDB_FILENAME_LEN] = "\0"; + + strncpy(tname, pf->aname, TSDB_FILENAME_LEN); + strncpy(dest, dirname(tname), TSDB_FILENAME_LEN); +} + +// DIR APIs ==================================== +int tfsMkdirAt(const char *rname, int level, int id) { + SDisk *pDisk = TFS_DISK_AT(level, id); + char aname[TMPNAME_LEN]; + + snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); + if (taosMkDir(aname, 0755) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +int tfsMkdirRecurAt(const char *rname, int level, int id) { + if (tfsMkdirAt(rname, level, id) < 0) { + if (errno == ENOENT) { + // Try to create upper + char *s = strdup(rname); + + if (tfsMkdirRecurAt(dirname(s), level, id) < 0) { + tfree(s); + return -1; + } + tfree(s); + + if (tfsMkdirAt(rname, level, id) < 0) { + return -1; + } + } else { + return -1; + } + } + + return 0; +} + +int tfsMkdir(const char *rname) { + for (int level = 0; level < TFS_NLEVEL(); level++) { + STier *pTier = TFS_TIER_AT(level); + for (int id = 0; id < TIER_NDISKS(pTier); id++) { + if (tfsMkdirAt(rname, level, id) < 0) { + return -1; + } + } + } + + return 0; +} + +int tfsRmdir(const char *rname) { + char aname[TMPNAME_LEN] = "\0"; + + for (int level = 0; level < TFS_NLEVEL(); level++) { + STier *pTier = TFS_TIER_AT(level); + for (int id = 0; id < TIER_NDISKS(pTier); id++) { + SDisk *pDisk = DISK_AT_TIER(pTier, id); + + snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); + + taosRemoveDir(aname); + } + } + + return 0; +} + +int tfsRename(char *orname, char *nrname) { + char oaname[TMPNAME_LEN] = "\0"; + char naname[TMPNAME_LEN] = "\0"; + + for (int level = 0; level < pfs->nlevel; level++) { + STier *pTier = TFS_TIER_AT(level); + for (int id = 0; id < TIER_NDISKS(pTier); id++) { + SDisk *pDisk = DISK_AT_TIER(pTier, id); + + snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname); + snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname); + + taosRename(oaname, naname); + } + } + + return 0; +} + +struct TDIR { + SDiskIter iter; + int level; + int id; + char dirname[TSDB_FILENAME_LEN]; + TFILE tfile; + DIR * dir; +}; + +TDIR *tfsOpendir(const char *rname) { + TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir)); + if (tdir == NULL) { + terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + return NULL; + } + + tfsInitDiskIter(&(tdir->iter)); + strncpy(tdir->dirname, rname, TSDB_FILENAME_LEN); + + if (tfsOpendirImpl(tdir) < 0) { + free(tdir); + return NULL; + } + + return tdir; +} + +const TFILE *tfsReaddir(TDIR *tdir) { + if (tdir == NULL || tdir->dir == NULL) return NULL; + char bname[TMPNAME_LEN * 2] = "\0"; + + while (true) { + struct dirent *dp = NULL; + dp = readdir(tdir->dir); + if (dp != NULL) { + // Skip . and .. + if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; + + snprintf(bname, TMPNAME_LEN * 2, "%s/%s", tdir->dirname, dp->d_name); + tfsInitFile(&(tdir->tfile), tdir->level, tdir->id, bname); + return &(tdir->tfile); + } + + if (tfsOpendirImpl(tdir) < 0) { + return NULL; + } + + if (tdir->dir == NULL) { + terrno = TSDB_CODE_SUCCESS; + return NULL; + } + } +} + +void tfsClosedir(TDIR *tdir) { + if (tdir) { + if (tdir->dir != NULL) { + closedir(tdir->dir); + tdir->dir = NULL; + } + free(tdir); + } +} + +// private +static int tfsMount(SDiskCfg *pCfg) { + SDiskID did; + SDisk * pDisk = NULL; + + if (tfsCheckAndFormatCfg(pCfg) < 0) return -1; + + did.level = pCfg->level; + pDisk = tfsMountDiskToTier(TFS_TIER_AT(did.level), pCfg); + if (pDisk == NULL) { + fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); + return -1; + } + did.id = DISK_ID(pDisk); + + taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did)); + if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1; + + return 0; +} + +static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) { + char dirName[TSDB_FILENAME_LEN] = "\0"; + struct stat pstat; + + if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) { + fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (pCfg->primary) { + if (pCfg->level != 0) { + fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (TFS_PRIMARY_DISK() != NULL) { + fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir); + terrno = TSDB_CODE_FS_DUP_PRIMARY; + return -1; + } + } + + if (tfsFormatDir(pCfg->dir, dirName) < 0) { + fError("failed to mount %s to FS since invalid dir format", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (tfsGetDiskByName(dirName) != NULL) { + fError("failed to mount %s to FS since duplicate mount", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (access(dirName, W_OK | R_OK | F_OK) != 0) { + fError("failed to mount %s to FS since no R/W access rights", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (stat(dirName, &pstat) < 0) { + fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (!S_ISDIR(pstat.st_mode)) { + fError("failed to mount %s to FS since not a directory", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN); + + return 0; +} + +static int tfsFormatDir(char *idir, char *odir) { + wordexp_t wep = {0}; + + int code = wordexp(idir, &wep, 0); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + if (realpath(wep.we_wordv[0], odir) == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wordfree(&wep); + return -1; + } + + wordfree(&wep); + return 0; + +} + +static int tfsCheck() { + if (TFS_PRIMARY_DISK() == NULL) { + fError("no primary disk is set"); + terrno = TSDB_CODE_FS_NO_PRIMARY_DISK; + return -1; + } + + for (int level = 0; level < TFS_NLEVEL(); level++) { + if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) { + fError("no disk at level %d", level); + terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER; + return -1; + } + } + + return 0; +} + +static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); } +static SDisk *tfsGetDiskByName(const char *dir) { + SDiskID did; + SDisk * pDisk = NULL; + void * pr = NULL; + + pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN)); + if (pr == NULL) return NULL; + + did = *(SDiskID *)pr; + pDisk = tfsGetDiskByID(did); + ASSERT(pDisk != NULL); + + return pDisk; +} + +static int tfsOpendirImpl(TDIR *tdir) { + SDisk *pDisk = NULL; + char adir[TMPNAME_LEN * 2] = "\0"; + + if (tdir->dir != NULL) { + closedir(tdir->dir); + tdir->dir = NULL; + } + + while (true) { + pDisk = tfsNextDisk(&(tdir->iter)); + if (pDisk == NULL) return 0; + + tdir->level = DISK_LEVEL(pDisk); + tdir->id = DISK_ID(pDisk); + + snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname); + tdir->dir = opendir(adir); + if (tdir->dir != NULL) break; + } + + return 0; +} + +static void tfsInitDiskIter(SDiskIter *pIter) { pIter->pDisk = TFS_DISK_AT(0, 0); } + +static SDisk *tfsNextDisk(SDiskIter *pIter) { + SDisk *pDisk = pIter->pDisk; + + if (pDisk == NULL) return NULL; + + int level = DISK_LEVEL(pDisk); + int id = DISK_ID(pDisk); + + id++; + if (id < TIER_NDISKS(TFS_TIER_AT(level))) { + pIter->pDisk = TFS_DISK_AT(level, id); + ASSERT(pIter->pDisk != NULL); + } else { + level++; + id = 0; + if (level < TFS_NLEVEL()) { + pIter->pDisk = TFS_DISK_AT(level, id); + ASSERT(pIter->pDisk != NULL); + } else { + pIter->pDisk = NULL; + } + } + + return pDisk; +} + +// OTHER FUNCTIONS =================================== +void taosGetDisk() { + const double unit = 1024 * 1024 * 1024; + SysDiskSize diskSize; + SFSMeta fsMeta; + + if (tscEmbedded) { + tfsUpdateInfo(&fsMeta); + tsTotalDataDirGB = (float)(fsMeta.tsize / unit); + tsAvailDataDirGB = (float)(fsMeta.avail / unit); + } + + if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { + tsTotalLogDirGB = (float)(diskSize.tsize / unit); + tsAvailLogDirGB = (float)(diskSize.avail / unit); + } + + if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { + tsTotalTmpDirGB = (float)(diskSize.tsize / unit); + tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); + } +} diff --git a/src/tfs/src/ttier.c b/src/tfs/src/ttier.c new file mode 100644 index 0000000000000000000000000000000000000000..2dce0c31949a145bdafd111a63e96c2b83a7140c --- /dev/null +++ b/src/tfs/src/ttier.c @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" + +#include "taosdef.h" +#include "taoserror.h" +#include "tfsint.h" + +#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock)) +#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock)) + +// PROTECTED ========================================== +int tfsInitTier(STier *pTier, int level) { + memset((void *)pTier, 0, sizeof(*pTier)); + + int code = pthread_spin_init(&(pTier->lock), 0); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + pTier->level = level; + + return 0; +} + +void tfsDestroyTier(STier *pTier) { + for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) { + DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id)); + } + + pTier->ndisk = 0; + + pthread_spin_destroy(&(pTier->lock)); +} + +SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { + ASSERT(pTier->level == pCfg->level); + + int id = 0; + SDisk *pDisk; + + if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) { + terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; + return NULL; + } + + if (pTier->level == 0) { + if (DISK_AT_TIER(pTier, 0) != NULL) { + id = pTier->ndisk; + } else { + if (pCfg->primary) { + id = 0; + } else { + id = pTier->ndisk + 1; + } + if (id >= TSDB_MAX_DISKS_PER_TIER) { + terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; + return NULL; + } + } + } else { + id = pTier->ndisk; + } + + pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); + if (pDisk == NULL) return NULL; + DISK_AT_TIER(pTier, id) = pDisk; + pTier->ndisk++; + + fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); + + return DISK_AT_TIER(pTier, id); +} + +void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) { + STierMeta tmeta; + + if (pTierMeta == NULL) { + pTierMeta = &tmeta; + } + memset(pTierMeta, 0, sizeof(*pTierMeta)); + + tfsLockTier(pTier); + + for (int id = 0; id < pTier->ndisk; id++) { + if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) { + continue; + } + pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id)); + pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id)); + pTierMeta->nAvailDisks++; + } + + pTier->tmeta = *pTierMeta; + + tfsUnLockTier(pTier); +} + +// Round-Robin to allocate disk on a tier +int tfsAllocDiskOnTier(STier *pTier) { + ASSERT(pTier->ndisk > 0); + int id = TFS_UNDECIDED_ID; + SDisk *pDisk; + + tfsLockTier(pTier); + + if (TIER_AVAIL_DISKS(pTier) <= 0) { + tfsUnLockTier(pTier); + return id; + } + + id = pTier->nextid; + while (true) { + pDisk = DISK_AT_TIER(pTier, id); + ASSERT(pDisk != NULL); + + if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) { + id = (id + 1) % pTier->ndisk; + if (id == pTier->nextid) { + tfsUnLockTier(pTier); + return TFS_UNDECIDED_ID; + } else { + continue; + } + } else { + pTier->nextid = (id + 1) % pTier->ndisk; + break; + } + } + + tfsUnLockTier(pTier); + return id; +} + +void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) { + ASSERT(pTierMeta != NULL); + + tfsLockTier(pTier); + *pTierMeta = pTier->tmeta; + tfsUnLockTier(pTier); +} + +void tfsPosNextId(STier *pTier) { + ASSERT(pTier->ndisk > 0); + int nextid = 0; + + for (int id = 1; id < pTier->ndisk; id++) { + SDisk *pLDisk = DISK_AT_TIER(pTier, nextid); + SDisk *pDisk = DISK_AT_TIER(pTier, id); + if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) { + nextid = id; + } + } + + pTier->nextid = nextid; +} \ No newline at end of file diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt index d86b104f234a7e39bd67f728370e188c965cf42a..31d52aae7d4a809044ab01a7b561801d1ad0c2eb 100644 --- a/src/tsdb/CMakeLists.txt +++ b/src/tsdb/CMakeLists.txt @@ -4,7 +4,7 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tsdb ${SRC}) -TARGET_LINK_LIBRARIES(tsdb common tutil) +TARGET_LINK_LIBRARIES(tsdb tfs common tutil) IF (TD_LINUX) # Someone has no gtest directory, so comment it diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h new file mode 100644 index 0000000000000000000000000000000000000000..414ace00097d95742080a8f173177d5e44497237 --- /dev/null +++ b/src/tsdb/inc/tsdbBuffer.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_BUFFER_H_ +#define _TD_TSDB_BUFFER_H_ + +typedef struct { + int64_t blockId; + int offset; + int remain; + char data[]; +} STsdbBufBlock; + +typedef struct { + pthread_cond_t poolNotEmpty; + int bufBlockSize; + int tBufBlocks; + int nBufBlocks; + int64_t index; + SList* bufBlockList; +} STsdbBufPool; + +#define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold + +STsdbBufPool* tsdbNewBufPool(); +void tsdbFreeBufPool(STsdbBufPool* pBufPool); +int tsdbOpenBufPool(STsdbRepo* pRepo); +void tsdbCloseBufPool(STsdbRepo* pRepo); +SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); + +#endif /* _TD_TSDB_BUFFER_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbCommit.h b/src/tsdb/inc/tsdbCommit.h new file mode 100644 index 0000000000000000000000000000000000000000..5e740081d187466435860a3a6c066412419ab571 --- /dev/null +++ b/src/tsdb/inc/tsdbCommit.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_COMMIT_H_ +#define _TD_TSDB_COMMIT_H_ + +typedef struct { + int minFid; + int midFid; + int maxFid; + TSKEY minKey; +} SRtn; + +typedef struct { + uint64_t uid; + int64_t offset; + int64_t size; +} SKVRecord; + +void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn); +int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord); +void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord); +void *tsdbCommitData(STsdbRepo *pRepo); + +static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { + if (fid >= pRtn->maxFid) { + return 0; + } else if (fid >= pRtn->midFid) { + return 1; + } else if (fid >= pRtn->minFid) { + return 2; + } else { + return -1; + } +} + +#endif /* _TD_TSDB_COMMIT_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbCommitQueue.h b/src/tsdb/inc/tsdbCommitQueue.h new file mode 100644 index 0000000000000000000000000000000000000000..c2353391f9ad88185097eb52907e890c96021f8c --- /dev/null +++ b/src/tsdb/inc/tsdbCommitQueue.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_COMMIT_QUEUE_H_ +#define _TD_TSDB_COMMIT_QUEUE_H_ + +int tsdbScheduleCommit(STsdbRepo *pRepo); + +#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbFS.h b/src/tsdb/inc/tsdbFS.h new file mode 100644 index 0000000000000000000000000000000000000000..d63aeb14ac6ca6cd6f59654cf74f11d8e33d6ce4 --- /dev/null +++ b/src/tsdb/inc/tsdbFS.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_FS_H_ +#define _TD_TSDB_FS_H_ + +#define TSDB_FS_VERSION 0 + +// ================== CURRENT file header info +typedef struct { + uint32_t version; // Current file system version (relating to code) + uint32_t len; // Encode content length (including checksum) +} SFSHeader; + +// ================== TSDB File System Meta +typedef struct { + uint32_t version; // Commit version from 0 to increase + int64_t totalPoints; // total points + int64_t totalStorage; // Uncompressed total storage +} STsdbFSMeta; + +// ================== +typedef struct { + STsdbFSMeta meta; // FS meta + SMFile* pmf; // meta file pointer + SMFile mf; // meta file + SArray* df; // data file array +} SFSStatus; + +typedef struct { + pthread_rwlock_t lock; + + SFSStatus* cstatus; // current status + SHashObj* metaCache; // meta cache + bool intxn; + SFSStatus* nstatus; // new status +} STsdbFS; + +#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus) +#define FS_NEW_STATUS(pfs) ((pfs)->nstatus) +#define FS_IN_TXN(pfs) (pfs)->intxn +#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version) +#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version) + +typedef struct { + int direction; + uint64_t version; // current FS version + STsdbFS* pfs; + int index; // used to position next fset when version the same + int fid; // used to seek when version is changed + SDFileSet* pSet; +} SFSIter; + +#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC +#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC + +STsdbFS *tsdbNewFS(STsdbCfg *pCfg); +void * tsdbFreeFS(STsdbFS *pfs); +int tsdbOpenFS(STsdbRepo *pRepo); +void tsdbCloseFS(STsdbRepo *pRepo); +void tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd); +int tsdbEndFSTxn(STsdbRepo *pRepo); +int tsdbEndFSTxnWithError(STsdbFS *pfs); +void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); +void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); +int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); + +void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); +void tsdbFSIterSeek(SFSIter *pIter, int fid); +SDFileSet *tsdbFSIterNext(SFSIter *pIter); +int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta); + +static FORCE_INLINE int tsdbRLockFS(STsdbFS* pFs) { + int code = pthread_rwlock_rdlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbWLockFS(STsdbFS* pFs) { + int code = pthread_rwlock_wrlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) { + int code = pthread_rwlock_unlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +#endif /* _TD_TSDB_FS_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h new file mode 100644 index 0000000000000000000000000000000000000000..f1e2422e45b195e3874518132051ded67a776449 --- /dev/null +++ b/src/tsdb/inc/tsdbFile.h @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TS_TSDB_FILE_H_ +#define _TS_TSDB_FILE_H_ + +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF +#define TSDB_IVLD_FID INT_MIN +#define TSDB_FILE_STATE_OK 0 +#define TSDB_FILE_STATE_BAD 1 + +#define TSDB_FILE_INFO(tf) (&((tf)->info)) +#define TSDB_FILE_F(tf) (&((tf)->f)) +#define TSDB_FILE_FD(tf) ((tf)->fd) +#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf)) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) +#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) +#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf)) +#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf)) +#define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf)) +#define TSDB_FILE_STATE(tf) ((tf)->state) +#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) +#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) +#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) + +typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T; + +// =============== SMFile +typedef struct { + int64_t size; + int64_t tombSize; + int64_t nRecords; + int64_t nDels; + uint32_t magic; +} SMFInfo; + +typedef struct { + SMFInfo info; + TFILE f; + int fd; + uint8_t state; +} SMFile; + +void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); +void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); +int tsdbEncodeSMFile(void** buf, SMFile* pMFile); +void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); +int tsdbEncodeSMFileEx(void** buf, SMFile* pMFile); +void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile); +int tsdbApplyMFileChange(SMFile* from, SMFile* to); +int tsdbCreateMFile(SMFile* pMFile, bool updateHeader); +int tsdbUpdateMFileHeader(SMFile* pMFile); +int tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo); +int tsdbScanAndTryFixMFile(STsdbRepo* pRepo); +int tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo); +void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo); + +static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; } + +static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) { + ASSERT(TSDB_FILE_CLOSED(pMFile)); + + pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags); + if (pMFile->fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) { + if (TSDB_FILE_OPENED(pMFile)) { + close(pMFile->fd); + TSDB_FILE_SET_CLOSED(pMFile); + } +} + +static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) { + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM)); +} + +static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t toffset; + + if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pMFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) { + return -1; + } + + pMFile->info.size += nbyte; + + return (int)nbyte; +} + +static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); } + +static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t nread = taosRead(pMFile->fd, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +// =============== SDFile +typedef struct { + uint32_t magic; + uint32_t len; + uint32_t totalBlocks; + uint32_t totalSubBlocks; + uint32_t offset; + uint64_t size; + uint64_t tombSize; +} SDFInfo; + +typedef struct { + SDFInfo info; + TFILE f; + int fd; + uint8_t state; +} SDFile; + +void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype); +void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); +int tsdbEncodeSDFile(void** buf, SDFile* pDFile); +void* tsdbDecodeSDFile(void* buf, SDFile* pDFile); +int tsdbCreateDFile(SDFile* pDFile, bool updateHeader); +int tsdbUpdateDFileHeader(SDFile* pDFile); +int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); +int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); + +static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; } + +static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { + ASSERT(!TSDB_FILE_OPENED(pDFile)); + + pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { + if (TSDB_FILE_OPENED(pDFile)) { + close(pDFile->fd); + TSDB_FILE_SET_CLOSED(pDFile); + } +} + +static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) { + pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM)); +} + +static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t toffset; + + if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pDFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { + return -1; + } + + pDFile->info.size += nbyte; + + return (int)nbyte; +} + +static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); } + +static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nread = taosRead(pDFile->fd, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { + if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc)); + return 0; +} + +// =============== SDFileSet +typedef struct { + int fid; + int state; + SDFile files[TSDB_FILE_MAX]; +} SDFileSet; + +#define TSDB_FSET_FID(s) ((s)->fid) +#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t)) +#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0)) +#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0)) +#define TSDB_FSET_SET_CLOSED(s) \ + do { \ + for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ + TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype)); \ + } \ + } while (0); +#define TSDB_FSET_FSYNC(s) \ + do { \ + for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ + TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype)); \ + } \ + } while (0); + +void tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver); +void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet); +int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet); +void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet); +int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet); +void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet); +int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to); +int tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader); +int tsdbUpdateDFileSetHeader(SDFileSet* pSet); +int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet); + +static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { + tsdbCloseDFileSet(pSet); + return -1; + } + } + return 0; +} + +static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { + tsdbRemoveDFileSet(pDest); + return -1; + } + } + + return 0; +} + +static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) { + *minKey = fid * days * tsMsPerDay[precision]; + *maxKey = *minKey + days * tsMsPerDay[precision] - 1; +} + +static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) { + return false; + } + } + + return true; +} + +#endif /* _TS_TSDB_FILE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbLog.h b/src/tsdb/inc/tsdbLog.h new file mode 100644 index 0000000000000000000000000000000000000000..fdd04e968a2ce5345030aa4cef2bb87becd8f3f9 --- /dev/null +++ b/src/tsdb/inc/tsdbLog.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_LOG_H_ +#define _TD_TSDB_LOG_H_ + +extern int32_t tsdbDebugFlag; + +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) + +#endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h deleted file mode 100644 index 5c978abd1da5cc5caa17450661d980ca648763c1..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbMain.h +++ /dev/null @@ -1,610 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_TSDB_MAIN_H_ -#define _TD_TSDB_MAIN_H_ - -#include "os.h" -#include "hash.h" -#include "tcoding.h" -#include "tglobal.h" -#include "tkvstore.h" -#include "tlist.h" -#include "tlog.h" -#include "tlockfree.h" -#include "tsdb.h" -#include "tskiplist.h" -#include "tutil.h" - -#ifdef __cplusplus -extern "C" { -#endif - -extern int tsdbDebugFlag; - -#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} -#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} -#define tsdbWarn(...) { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} -#define tsdbInfo(...) { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} -#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} -#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} - -#define TSDB_MAX_TABLE_SCHEMAS 16 -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F -#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF - -#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax))) - -// NOTE: Any file format change must increase this version number by 1 -// Also, implement the convert function -#define TSDB_FILE_VERSION ((uint32_t)0) - -// Definitions -// ------------------ tsdbMeta.c -typedef struct STable { - STableId tableId; - ETableType type; - tstr* name; // NOTE: there a flexible string here - uint64_t suid; - struct STable* pSuper; // super table pointer - uint8_t numOfSchemas; - STSchema* schema[TSDB_MAX_TABLE_SCHEMAS]; - STSchema* tagSchema; - SKVRow tagVal; - SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - void* eventHandler; // TODO - void* streamHandler; // TODO - TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure - char* sql; - void* cqhandle; - SRWLatch latch; // TODO: implementa latch functions - T_REF_DECLARE() -} STable; - -typedef struct { - pthread_rwlock_t rwLock; - - int32_t nTables; - int32_t maxTables; - STable** tables; - SList* superList; - SHashObj* uidMap; - SKVStore* pStore; - int maxRowBytes; - int maxCols; -} STsdbMeta; - -// ------------------ tsdbBuffer.c -typedef struct { - int64_t blockId; - int offset; - int remain; - char data[]; -} STsdbBufBlock; - -typedef struct { - pthread_cond_t poolNotEmpty; - int bufBlockSize; - int tBufBlocks; - int nBufBlocks; - int64_t index; - SList* bufBlockList; -} STsdbBufPool; - -// ------------------ tsdbMemTable.c -typedef struct { - STable * pTable; - SSkipListIterator *pIter; -} SCommitIter; - -typedef struct { - uint64_t uid; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - SSkipList* pData; -} STableData; - -typedef struct { - T_REF_DECLARE() - SRWLatch latch; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - int32_t maxTables; - STableData** tData; - SList* actList; - SList* extraBuffList; - SList* bufBlockList; -} SMemTable; - -enum { TSDB_UPDATE_META, TSDB_DROP_META }; - -#ifdef WINDOWS -#pragma pack(push ,1) -typedef struct { -#else -typedef struct __attribute__((packed)){ -#endif - char act; - uint64_t uid; -} SActObj; -#ifdef WINDOWS -#pragma pack(pop) -#endif - -typedef struct { - int len; - char cont[]; -} SActCont; - -// ------------------ tsdbFile.c -extern const char* tsdbFileSuffix[]; -typedef enum { - TSDB_FILE_TYPE_HEAD = 0, - TSDB_FILE_TYPE_DATA, - TSDB_FILE_TYPE_LAST, - TSDB_FILE_TYPE_STAT, - TSDB_FILE_TYPE_NHEAD, - TSDB_FILE_TYPE_NDATA, - TSDB_FILE_TYPE_NLAST, - TSDB_FILE_TYPE_NSTAT -} TSDB_FILE_TYPE; - -#ifndef TDINTERNAL -#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1) -#else -#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1) -#endif - -typedef struct { - uint32_t magic; - uint32_t len; - uint32_t totalBlocks; - uint32_t totalSubBlocks; - uint32_t offset; - uint64_t size; // total size of the file - uint64_t tombSize; // unused file size -} STsdbFileInfo; - -typedef struct { - char fname[TSDB_FILENAME_LEN]; - int fd; - - STsdbFileInfo info; -} SFile; - -typedef struct { - int fileId; - int state; // 0 for health, 1 for problem - SFile files[TSDB_FILE_TYPE_MAX]; -} SFileGroup; - -typedef struct { - pthread_rwlock_t fhlock; - - int maxFGroups; - int nFGroups; - SFileGroup* pFGroup; -} STsdbFileH; - -typedef struct { - int direction; - STsdbFileH* pFileH; - int fileId; - int index; -} SFileGroupIter; - -// ------------------ tsdbMain.c -typedef struct { - int32_t totalLen; - int32_t len; - SDataRow row; -} SSubmitBlkIter; - -typedef struct { - int32_t totalLen; - int32_t len; - void * pMsg; -} SSubmitMsgIter; - -typedef struct { - int8_t state; - - char* rootDir; - STsdbCfg config; - STsdbAppH appH; - STsdbStat stat; - STsdbMeta* tsdbMeta; - STsdbBufPool* pPool; - SMemTable* mem; - SMemTable* imem; - STsdbFileH* tsdbFileH; - sem_t readyToCommit; - pthread_mutex_t mutex; - bool repoLocked; - int32_t code; // Commit code -} STsdbRepo; - -// ------------------ tsdbRWHelper.c -typedef struct { - int32_t tid; - uint32_t len; - uint32_t offset; - uint32_t hasLast : 2; - uint32_t numOfBlocks : 30; - uint64_t uid; - TSKEY maxKey; -} SCompIdx; - -typedef struct { - int64_t last : 1; - int64_t offset : 63; - int32_t algorithm : 8; - int32_t numOfRows : 24; - int32_t len; - int32_t keyLen; // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols - int16_t numOfSubBlocks; - int16_t numOfCols; // not including timestamp column - TSKEY keyFirst; - TSKEY keyLast; -} SCompBlock; - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t tid; - uint64_t uid; - SCompBlock blocks[]; -} SCompInfo; - -typedef struct { - int16_t colId; - int32_t len; - int32_t type : 8; - int32_t offset : 24; - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIndex; - int16_t minIndex; - int16_t numOfNull; - char padding[2]; -} SCompCol; - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t numOfCols; // For recovery usage - uint64_t uid; // For recovery usage - SCompCol cols[]; -} SCompData; - -typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; - -typedef struct { - TSKEY minKey; - TSKEY maxKey; - SFileGroup fGroup; - SFile nHeadF; - SFile nLastF; -} SHelperFile; - -typedef struct { - uint64_t uid; - int32_t tid; -} SHelperTable; - -typedef struct { - SCompIdx* pIdxArray; - int numOfIdx; - int curIdx; -} SIdxH; - -typedef struct { - tsdb_rw_helper_t type; - - STsdbRepo* pRepo; - int8_t state; - // For file set usage - SHelperFile files; - SIdxH idxH; - SCompIdx curCompIdx; - void* pWIdx; - // For table set usage - SHelperTable tableInfo; - SCompInfo* pCompInfo; - bool hasOldLastBlock; - // For block set usage - SCompData* pCompData; - SDataCols* pDataCols[2]; - void* pBuffer; // Buffer to hold the whole data block - void* compBuffer; // Buffer for temperary compress/decompress purpose -} SRWHelper; - -typedef struct { - int rowsInserted; - int rowsUpdated; - int rowsDeleteSucceed; - int rowsDeleteFailed; - int nOperations; - TSKEY keyFirst; - TSKEY keyLast; -} SMergeInfo; -// ------------------ tsdbScan.c -typedef struct { - SFileGroup fGroup; - int numOfIdx; - SCompIdx* pCompIdx; - SCompInfo* pCompInfo; - void* pBuf; - FILE* tLogStream; -} STsdbScanHandle; - -// Operations -// ------------------ tsdbMeta.c -#define TSDB_INIT_NTABLES 1024 -#define TABLE_TYPE(t) (t)->type -#define TABLE_NAME(t) (t)->name -#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data -#define TABLE_UID(t) (t)->tableId.uid -#define TABLE_TID(t) (t)->tableId.tid -#define TABLE_SUID(t) (t)->suid -#define TABLE_LASTKEY(t) (t)->lastKey -#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) - -STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); -void tsdbFreeMeta(STsdbMeta* pMeta); -int tsdbOpenMeta(STsdbRepo* pRepo); -int tsdbCloseMeta(STsdbRepo* pRepo); -STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); -STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version); -int tsdbWLockRepoMeta(STsdbRepo* pRepo); -int tsdbRLockRepoMeta(STsdbRepo* pRepo); -int tsdbUnlockRepoMeta(STsdbRepo* pRepo); -void tsdbRefTable(STable* pTable); -void tsdbUnRefTable(STable* pTable); -void tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct); - -static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { - if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { - return -1; - } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) { - return 1; - } else { - return 0; - } -} - -static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) { - STable* pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; - STSchema* pSchema = NULL; - STSchema* pTSchema = NULL; - - if (lock) taosRLockLatch(&(pDTable->latch)); - if (version < 0) { // get the latest version of schema - pTSchema = pDTable->schema[pDTable->numOfSchemas - 1]; - } else { // get the schema with version - void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*), - tsdbCompareSchemaVersion, TD_EQ); - if (ptr == NULL) { - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - goto _exit; - } - pTSchema = *(STSchema**)ptr; - } - - ASSERT(pTSchema != NULL); - - if (copy) { - if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - } else { - pSchema = pTSchema; - } - -_exit: - if (lock) taosRUnLockLatch(&(pDTable->latch)); - return pSchema; -} - -static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) { - return tsdbGetTableSchemaImpl(pTable, false, false, -1); -} - -static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { - if (pTable->type == TSDB_CHILD_TABLE) { // check child table first - STable *pSuper = pTable->pSuper; - if (pSuper == NULL) return NULL; - return pSuper->tagSchema; - } else if (pTable->type == TSDB_SUPER_TABLE) { - return pTable->tagSchema; - } else { - return NULL; - } -} - -// ------------------ tsdbBuffer.c -#define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold - -STsdbBufPool* tsdbNewBufPool(); -void tsdbFreeBufPool(STsdbBufPool* pBufPool); -int tsdbOpenBufPool(STsdbRepo* pRepo); -void tsdbCloseBufPool(STsdbRepo* pRepo); -SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); - -// ------------------ tsdbMemTable.c -int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); -void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem); -void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); -int tsdbAsyncCommit(STsdbRepo* pRepo); -int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); -void* tsdbCommitData(STsdbRepo* pRepo); - -static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { - if (pIter == NULL) return NULL; - - SSkipListNode* node = tSkipListIterGet(pIter); - if (node == NULL) return NULL; - - return (SDataRow)SL_GET_NODE_DATA(node); -} - -static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; - - return dataRowKey(row); -} - -static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TKEY_NULL; - - return dataRowTKey(row); -} - -static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { - ASSERT(pRepo != NULL); - if (pRepo->mem == NULL) return NULL; - - SListNode* pNode = listTail(pRepo->mem->bufBlockList); - if (pNode == NULL) return NULL; - - STsdbBufBlock* pBufBlock = NULL; - tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock)); - - return pBufBlock; -} - -// ------------------ tsdbFile.c -#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) -#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3) -#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId -#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId -#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0) -#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC -#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC - -STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); -void tsdbFreeFileH(STsdbFileH* pFileH); -int tsdbOpenFileH(STsdbRepo* pRepo); -void tsdbCloseFileH(STsdbRepo* pRepo); -SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid); -void tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction); -void tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid); -SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter); -int tsdbOpenFile(SFile* pFile, int oflag); -void tsdbCloseFile(SFile* pFile); -int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type); -SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags); -void tsdbFitRetention(STsdbRepo* pRepo); -int tsdbUpdateFileHeader(SFile* pFile); -int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); -void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); -void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); -int tsdbLoadFileHeader(SFile* pFile, uint32_t* version); -void tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size); -void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); - -// ------------------ tsdbRWHelper.c -#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state -#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set -#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded -#define TSDB_HELPER_TABLE_SET 0x4 // Table is set -#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded -#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded -#define helperSetState(h, s) (((h)->state) |= (s)) -#define helperClearState(h, s) ((h)->state &= (~(s))) -#define helperHasState(h, s) ((((h)->state) & (s)) == (s)) -#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx) -#define TSDB_MAX_SUBBLOCKS 8 -#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) -#define helperType(h) (h)->type -#define helperRepo(h) (h)->pRepo -#define helperState(h) (h)->state -#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) -#define helperFileId(h) ((h)->files.fGroup.fileId) -#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) -#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) -#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) -#define helperNewHeadF(h) (&((h)->files.nHeadF)) -#define helperNewLastF(h) (&((h)->files.nLastF)) - -int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo); -int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); -void tsdbDestroyHelper(SRWHelper* pHelper); -void tsdbResetHelper(SRWHelper* pHelper); -int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); -int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup); -int tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); -int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey); -int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); -int tsdbWriteCompInfo(SRWHelper* pHelper); -int tsdbWriteCompIdx(SRWHelper* pHelper); -int tsdbLoadCompIdxImpl(SFile* pFile, uint32_t offset, uint32_t len, void* buffer); -int tsdbDecodeSCompIdxImpl(void* buffer, uint32_t len, SCompIdx** ppCompIdx, int* numOfIdx); -int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); -int tsdbLoadCompInfoImpl(SFile* pFile, SCompIdx* pIdx, SCompInfo** ppCompInfo); -int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); -int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); -void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); -int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds, - int numOfColIds); -int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo); - -static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) { - if (*(TSKEY*)key1 > *(TSKEY*)key2) { - return 1; - } else if (*(TSKEY*)key1 == *(TSKEY*)key2) { - return 0; - } else { - return -1; - } -} - -// ------------------ tsdbMain.c -#define REPO_ID(r) (r)->config.tsdbId -#define IS_REPO_LOCKED(r) (r)->repoLocked -#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) - -char* tsdbGetMetaFileName(char* rootDir); -void tsdbGetDataFileName(char* rootDir, int vid, int fid, int type, char* fname); -int tsdbLockRepo(STsdbRepo* pRepo); -int tsdbUnlockRepo(STsdbRepo* pRepo); -char* tsdbGetDataDirName(char* rootDir); -int tsdbGetNextMaxTables(int tid); -STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo); -STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo); -int tsdbCheckCommit(STsdbRepo* pRepo); - -// ------------------ tsdbScan.c -int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid); -STsdbScanHandle* tsdbNewScanHandle(); -void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream); -int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid); -int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle); -int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx); -int tsdbCloseScanFile(STsdbScanHandle* pScanHandle); -void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle); - -// ------------------ tsdbCommitQueue.c -int tsdbScheduleCommit(STsdbRepo *pRepo); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h new file mode 100644 index 0000000000000000000000000000000000000000..bd64ed4a5238b3b8b60716d8732d59d27218c639 --- /dev/null +++ b/src/tsdb/inc/tsdbMemTable.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_MEMTABLE_H_ +#define _TD_TSDB_MEMTABLE_H_ + +typedef struct { + int rowsInserted; + int rowsUpdated; + int rowsDeleteSucceed; + int rowsDeleteFailed; + int nOperations; + TSKEY keyFirst; + TSKEY keyLast; +} SMergeInfo; + +typedef struct { + STable * pTable; + SSkipListIterator *pIter; +} SCommitIter; + +typedef struct { + uint64_t uid; + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfRows; + SSkipList* pData; + T_REF_DECLARE() +} STableData; + +typedef struct { + T_REF_DECLARE() + SRWLatch latch; + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfRows; + int32_t maxTables; + STableData** tData; + SList* actList; + SList* extraBuffList; + SList* bufBlockList; + int64_t pointsAdd; // TODO + int64_t storageAdd; // TODO +} SMemTable; + +enum { TSDB_UPDATE_META, TSDB_DROP_META }; + +#ifdef WINDOWS +#pragma pack(push ,1) +typedef struct { +#else +typedef struct __attribute__((packed)){ +#endif + char act; + uint64_t uid; +} SActObj; +#ifdef WINDOWS +#pragma pack(pop) +#endif + +typedef struct { + int len; + char cont[]; +} SActCont; + +int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem, SArray* pATable); +void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem); +void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); +int tsdbAsyncCommit(STsdbRepo* pRepo); +int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, + TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); +void* tsdbCommitData(STsdbRepo* pRepo); + +static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { + if (pIter == NULL) return NULL; + + SSkipListNode* node = tSkipListIterGet(pIter); + if (node == NULL) return NULL; + + return (SDataRow)SL_GET_NODE_DATA(node); +} + +static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { + SDataRow row = tsdbNextIterRow(pIter); + if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; + + return dataRowKey(row); +} + +static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { + SDataRow row = tsdbNextIterRow(pIter); + if (row == NULL) return TKEY_NULL; + + return dataRowTKey(row); +} + +#endif /* _TD_TSDB_MEMTABLE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h new file mode 100644 index 0000000000000000000000000000000000000000..cc916fa689a89f96e36c4419f1b2d413516ed5fc --- /dev/null +++ b/src/tsdb/inc/tsdbMeta.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_META_H_ +#define _TD_TSDB_META_H_ + +#define TSDB_MAX_TABLE_SCHEMAS 16 + +typedef struct STable { + STableId tableId; + ETableType type; + tstr* name; // NOTE: there a flexible string here + uint64_t suid; + struct STable* pSuper; // super table pointer + uint8_t numOfSchemas; + STSchema* schema[TSDB_MAX_TABLE_SCHEMAS]; + STSchema* tagSchema; + SKVRow tagVal; + SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index + void* eventHandler; // TODO + void* streamHandler; // TODO + TSKEY lastKey; + SDataRow lastRow; + char* sql; + void* cqhandle; + SRWLatch latch; // TODO: implementa latch functions + T_REF_DECLARE() +} STable; + +typedef struct { + pthread_rwlock_t rwLock; + + int32_t nTables; + int32_t maxTables; + STable** tables; + SList* superList; + SHashObj* uidMap; + int maxRowBytes; + int maxCols; +} STsdbMeta; + +#define TSDB_INIT_NTABLES 1024 +#define TABLE_TYPE(t) (t)->type +#define TABLE_NAME(t) (t)->name +#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data +#define TABLE_UID(t) (t)->tableId.uid +#define TABLE_TID(t) (t)->tableId.tid +#define TABLE_SUID(t) (t)->suid +// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) +#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch)) +#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch)) +#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch)) +#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch)) + +STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); +void tsdbFreeMeta(STsdbMeta* pMeta); +int tsdbOpenMeta(STsdbRepo* pRepo); +int tsdbCloseMeta(STsdbRepo* pRepo); +STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); +STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version); +int tsdbWLockRepoMeta(STsdbRepo* pRepo); +int tsdbRLockRepoMeta(STsdbRepo* pRepo); +int tsdbUnlockRepoMeta(STsdbRepo* pRepo); +void tsdbRefTable(STable* pTable); +void tsdbUnRefTable(STable* pTable); +void tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct); +int tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen); +void tsdbOrgMeta(STsdbRepo* pRepo); + +static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { + if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { + return -1; + } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) { + return 1; + } else { + return 0; + } +} + +static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) { + STable* pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; + STSchema* pSchema = NULL; + STSchema* pTSchema = NULL; + + if (lock) TSDB_RLOCK_TABLE(pDTable); + if (version < 0) { // get the latest version of schema + pTSchema = pDTable->schema[pDTable->numOfSchemas - 1]; + } else { // get the schema with version + void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*), + tsdbCompareSchemaVersion, TD_EQ); + if (ptr == NULL) { + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + goto _exit; + } + pTSchema = *(STSchema**)ptr; + } + + ASSERT(pTSchema != NULL); + + if (copy) { + if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + } else { + pSchema = pTSchema; + } + +_exit: + if (lock) TSDB_RUNLOCK_TABLE(pDTable); + return pSchema; +} + +static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) { + return tsdbGetTableSchemaImpl(pTable, false, false, -1); +} + +static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { + if (pTable->type == TSDB_CHILD_TABLE) { // check child table first + STable *pSuper = pTable->pSuper; + if (pSuper == NULL) return NULL; + return pSuper->tagSchema; + } else if (pTable->type == TSDB_SUPER_TABLE) { + return pTable->tagSchema; + } else { + return NULL; + } +} + +static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { + ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); + return pTable->lastKey; +} + +#endif /* _TD_TSDB_META_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbReadImpl.h b/src/tsdb/inc/tsdbReadImpl.h new file mode 100644 index 0000000000000000000000000000000000000000..e7840d9723f7935c69f70f1c236a5ed49c82c146 --- /dev/null +++ b/src/tsdb/inc/tsdbReadImpl.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_READ_IMPL_H_ +#define _TD_TSDB_READ_IMPL_H_ + +typedef struct SReadH SReadH; + +typedef struct { + int32_t tid; + uint32_t len; + uint32_t offset; + uint32_t hasLast : 2; + uint32_t numOfBlocks : 30; + uint64_t uid; + TSKEY maxKey; +} SBlockIdx; + +typedef struct { + int64_t last : 1; + int64_t offset : 63; + int32_t algorithm : 8; + int32_t numOfRows : 24; + int32_t len; + int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols + int16_t numOfSubBlocks; + int16_t numOfCols; // not including timestamp column + TSKEY keyFirst; + TSKEY keyLast; +} SBlock; + +typedef struct { + int32_t delimiter; // For recovery usage + int32_t tid; + uint64_t uid; + SBlock blocks[]; +} SBlockInfo; + +typedef struct { + int16_t colId; + int32_t len; + uint32_t type : 8; + uint32_t offset : 24; + int64_t sum; + int64_t max; + int64_t min; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + uint8_t offsetH; + char padding[1]; +} SBlockCol; + +// Code here just for back-ward compatibility +static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { + pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); + pBlockCol->offsetH = (uint8_t)(offset >> 24); +} + +static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { + uint32_t offset1 = pBlockCol->offset; + uint32_t offset2 = pBlockCol->offsetH; + return (offset1 | (offset2 << 24)); +} + +typedef struct { + int32_t delimiter; // For recovery usage + int32_t numOfCols; // For recovery usage + uint64_t uid; // For recovery usage + SBlockCol cols[]; +} SBlockData; + +struct SReadH { + STsdbRepo * pRepo; + SDFileSet rSet; // FSET to read + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; // table to read + SBlockIdx * pBlkIdx; // current reading table SBlockIdx + int cidx; + SBlockInfo *pBlkInfo; + SBlockData *pBlkData; // Block info + SDataCols * pDCols[2]; + void * pBuf; // buffer + void * pCBuf; // compression buffer +}; + +#define TSDB_READ_REPO(rh) ((rh)->pRepo) +#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) +#define TSDB_READ_FSET(rh) (&((rh)->rSet)) +#define TSDB_READ_TABLE(rh) ((rh)->pTable) +#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) +#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) +#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) +#define TSDB_READ_BUF(rh) ((rh)->pBuf) +#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) + +#define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM)) + +int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo); +void tsdbDestroyReadH(SReadH *pReadh); +int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet); +void tsdbCloseAndUnsetFSet(SReadH *pReadh); +int tsdbLoadBlockIdx(SReadH *pReadh); +int tsdbSetReadTable(SReadH *pReadh, STable *pTable); +int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); +int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); +int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds); +int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); +int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); +void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols); + +static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { + void * pBuf = *ppBuf; + size_t tsize = taosTSizeof(pBuf); + + if (tsize < size) { + if (tsize == 0) tsize = 1024; + + while (tsize < size) { + tsize *= 2; + } + + *ppBuf = taosTRealloc(pBuf, tsize); + if (*ppBuf == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + return 0; +} + +#endif /*_TD_TSDB_READ_IMPL_H_*/ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h new file mode 100644 index 0000000000000000000000000000000000000000..074ff20f2298918f1fa0698be0a291081ead8f05 --- /dev/null +++ b/src/tsdb/inc/tsdbint.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_INT_H_ +#define _TD_TSDB_INT_H_ + +// // TODO: remove the include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include + +#include "os.h" +#include "tlog.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tchecksum.h" +#include "tskiplist.h" +#include "tdataformat.h" +#include "tcoding.h" +#include "tscompression.h" +#include "tlockfree.h" +#include "tlist.h" +#include "hash.h" +#include "tarray.h" +#include "tfs.h" +#include "tsocket.h" + +#include "tsdb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Log +#include "tsdbLog.h" +// Meta +#include "tsdbMeta.h" +// Buffer +#include "tsdbBuffer.h" +// MemTable +#include "tsdbMemTable.h" +// File +#include "tsdbFile.h" +// FS +#include "tsdbFS.h" +// ReadImpl +#include "tsdbReadImpl.h" +// Commit +#include "tsdbCommit.h" +// Commit Queue +#include "tsdbCommitQueue.h" +// Main definitions +struct STsdbRepo { + uint8_t state; + + STsdbCfg config; + STsdbAppH appH; + STsdbStat stat; + STsdbMeta* tsdbMeta; + STsdbBufPool* pPool; + SMemTable* mem; + SMemTable* imem; + STsdbFS* fs; + tsem_t readyToCommit; + pthread_mutex_t mutex; + bool repoLocked; + int32_t code; // Commit code +}; + +#define REPO_ID(r) (r)->config.tsdbId +#define REPO_CFG(r) (&((r)->config)) +#define REPO_FS(r) ((r)->fs) +#define IS_REPO_LOCKED(r) (r)->repoLocked +#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) + +int tsdbLockRepo(STsdbRepo* pRepo); +int tsdbUnlockRepo(STsdbRepo* pRepo); +STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo); +int tsdbCheckCommit(STsdbRepo* pRepo); +int tsdbRestoreInfo(STsdbRepo* pRepo); +void tsdbGetRootDir(int repoid, char dirName[]); +void tsdbGetDataDir(int repoid, char dirName[]); + +static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { + ASSERT(pRepo != NULL); + if (pRepo->mem == NULL) return NULL; + + SListNode* pNode = listTail(pRepo->mem->bufBlockList); + if (pNode == NULL) return NULL; + + STsdbBufBlock* pBufBlock = NULL; + tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock)); + + return pBufBlock; +} + +static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { + ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); + int maxTables = TSDB_INIT_NTABLES; + while (true) { + maxTables = MIN(maxTables, TSDB_MAX_TABLES); + if (tid <= maxTables) break; + maxTables *= 2; + } + + return maxTables + 1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _TD_TSDB_INT_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 7cea27658c80d689972e3cb0f5dda3269a34b720..1798a21b9963c7641dd99dc7fa11a5dd977e0e3c 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "tsdb.h" -#include "tsdbMain.h" +#include "tsdbint.h" #define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 637b02cd32ae8ad8e4077684609dcac23922d8a0..a4cc316725d60510441c4badb77ce9a2805c08ca 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -12,23 +12,79 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "tsdbMain.h" +#include "tsdbint.h" + +#define TSDB_MAX_SUBBLOCKS 8 +#define TSDB_KEY_FID(key, days, precision) ((key) / tsMsPerDay[(precision)] / (days)) + +typedef struct { + SRtn rtn; // retention snapshot + SFSIter fsIter; // tsdb file iterator + int niters; // memory iterators + SCommitIter *iters; + bool isRFileSet; // read and commit FSET + SReadH readh; + SDFileSet wSet; + bool isDFileSame; + bool isLFileSame; + TSKEY minKey; + TSKEY maxKey; + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; + SArray * aSupBlk; // Table super-block array + SArray * aSubBlk; // table sub-block array + SDataCols * pDataCols; +} SCommitH; + +#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) +#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) +#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) +#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) +#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) +#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) +#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) +#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) +#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) +#define TSDB_COMMIT_DEFAULT_ROWS(ch) (TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock * 4 / 5) +#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) -static int tsdbCommitTSData(STsdbRepo *pRepo); static int tsdbCommitMeta(STsdbRepo *pRepo); +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen); +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); +static int tsdbCommitTSData(STsdbRepo *pRepo); +static void tsdbStartCommit(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo, int eno); -static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); -static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); -static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static int tsdbCreateCommitIters(SCommitH *pCommith); +static void tsdbDestroyCommitIters(SCommitH *pCommith); +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo); +static void tsdbDestroyCommitH(SCommitH *pCommith); +static int tsdbGetFidLevel(int fid, SRtn *pRtn); +static int tsdbNextCommitFid(SCommitH *pCommith); +static int tsdbCommitToTable(SCommitH *pCommith, int tid); +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); +static int tsdbComparKeyBlock(const void *arg1, const void *arg2); +static int tsdbWriteBlockInfo(SCommitH *pCommih); +static int tsdbWriteBlockIdx(SCommitH *pCommih); +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); +static int tsdbMoveBlock(SCommitH *pCommith, int bidx); +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, + bool isLastOneBlock); +static void tsdbResetCommitFile(SCommitH *pCommith); +static void tsdbResetCommitTable(SCommitH *pCommith); +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update); +static int tsdbApplyRtn(STsdbRepo *pRepo); +static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn); void *tsdbCommitData(STsdbRepo *pRepo) { - SMemTable * pMem = pRepo->imem; - - tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", - REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); - - pRepo->code = TSDB_CODE_SUCCESS; + tsdbStartCommit(pRepo); // Commit to update meta file if (tsdbCommitMeta(pRepo) < 0) { @@ -42,299 +98,1385 @@ void *tsdbCommitData(STsdbRepo *pRepo) { goto _err; } - tsdbFitRetention(pRepo); - - tsdbInfo("vgId:%d commit over, succeed", REPO_ID(pRepo)); tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); - return NULL; _err: ASSERT(terrno != TSDB_CODE_SUCCESS); pRepo->code = terrno; - tsdbInfo("vgId:%d commit over, failed", REPO_ID(pRepo)); - tsdbEndCommit(pRepo, terrno); + tsdbEndCommit(pRepo, terrno); return NULL; } -static int tsdbCommitTSData(STsdbRepo *pRepo) { - SMemTable * pMem = pRepo->imem; - SDataCols * pDataCols = NULL; - STsdbMeta * pMeta = pRepo->tsdbMeta; - SCommitIter *iters = NULL; - SRWHelper whelper = {0}; - STsdbCfg * pCfg = &(pRepo->config); - - if (pMem->numOfRows <= 0) return 0; - - iters = tsdbCreateCommitIters(pRepo); - if (iters == NULL) { - tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; +// =================== Commit Meta Data +static int tsdbCommitMeta(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SMemTable *pMem = pRepo->imem; + SMFile * pOMFile = pfs->cstatus->pmf; + SMFile mf; + SActObj * pAct = NULL; + SActCont * pCont = NULL; + SListNode *pNode = NULL; + SDiskID did; + + ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); + + if (listNEles(pMem->actList) <= 0) { + // no meta data to commit, just keep the old meta file + tsdbUpdateMFile(pfs, pOMFile); + return 0; + } else { + // Create/Open a meta file or open the existing file + if (pOMFile == NULL) { + // Create a new meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); + } else { + tsdbInitMFileEx(&mf, pOMFile); + if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } } - if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { - tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + // Loop to write + while ((pNode = tdListPopHead(pMem->actList)) != NULL) { + pAct = (SActObj *)pNode->data; + if (pAct->act == TSDB_UPDATE_META) { + pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); + if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else if (pAct->act == TSDB_DROP_META) { + if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { + tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else { + ASSERT(false); + } } - if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", - REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); - goto _err; + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; } - int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision)); - int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision)); + TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + tsdbUpdateMFile(pfs, &mf); - // Loop to commit to each file - for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { - tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - goto _err; - } + return 0; +} + +int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pRecord->uid); + tlen += taosEncodeFixedI64(buf, pRecord->offset); + tlen += taosEncodeFixedI64(buf, pRecord->size); + + return tlen; +} + +void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { + buf = taosDecodeFixedU64(buf, &(pRecord->uid)); + buf = taosDecodeFixedI64(buf, &(pRecord->offset)); + buf = taosDecodeFixedI64(buf, &(pRecord->size)); + + return buf; +} + +void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + TSKEY minKey, midKey, maxKey, now; + + now = taosGetTimestamp(pCfg->precision); + minKey = now - pCfg->keep * tsMsPerDay[pCfg->precision]; + midKey = now - pCfg->keep2 * tsMsPerDay[pCfg->precision]; + maxKey = now - pCfg->keep1 * tsMsPerDay[pCfg->precision]; + + pRtn->minKey = minKey; + pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); + tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + pRtn->minFid, pRtn->midFid, pRtn->maxFid); +} + +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) { + char buf[64] = "\0"; + void * pBuf = buf; + SKVRecord rInfo; + int64_t offset; + + // Seek to end of meta file + offset = tsdbSeekMFile(pMFile, 0, SEEK_END); + if (offset < 0) { + return -1; + } + + rInfo.offset = offset; + rInfo.uid = uid; + rInfo.size = contLen; + + int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); + if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { + return -1; + } + + if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { + return -1; } - tdFreeDataCols(pDataCols); - tsdbDestroyCommitIters(iters, pMem->maxTables); - tsdbDestroyHelper(&whelper); + tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); + SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid)); + if (pRecord != NULL) { + pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); + } else { + pMFile->info.nRecords++; + } + taosHashPut(pfs->metaCache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); return 0; +} -_err: - tdFreeDataCols(pDataCols); - tsdbDestroyCommitIters(iters, pMem->maxTables); - tsdbDestroyHelper(&whelper); +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { + SKVRecord rInfo = {0}; + char buf[128] = "\0"; + + SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); + if (pRecord == NULL) { + tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); + return -1; + } + + rInfo.offset = -pRecord->offset; + rInfo.uid = pRecord->uid; + rInfo.size = pRecord->size; + + void *pBuf = buf; + tsdbEncodeKVRecord(&pBuf, &rInfo); + + if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { + return -1; + } - return -1; + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); + pMFile->info.nDels++; + pMFile->info.nRecords--; + pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); + + taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); + return 0; } -static int tsdbCommitMeta(STsdbRepo *pRepo) { +// =================== Commit Time-Series Data +static int tsdbCommitTSData(STsdbRepo *pRepo) { SMemTable *pMem = pRepo->imem; - STsdbMeta *pMeta = pRepo->tsdbMeta; - SActObj * pAct = NULL; - SActCont * pCont = NULL; + SCommitH commith; + SDFileSet *pSet = NULL; + int fid; - if (listNEles(pMem->actList) <= 0) return 0; + memset(&commith, 0, sizeof(SMemTable *)); - if (tdKVStoreStartCommit(pMeta->pStore) < 0) { - tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + if (pMem->numOfRows <= 0) { + // No memory data, just apply retention on each file on disk + if (tsdbApplyRtn(pRepo) < 0) { + return -1; + } + return 0; } - SListNode *pNode = NULL; + // Resource initialization + if (tsdbInitCommitH(&commith, pRepo) < 0) { + return -1; + } - while ((pNode = tdListPopHead(pMem->actList)) != NULL) { - pAct = (SActObj *)pNode->data; - if (pAct->act == TSDB_UPDATE_META) { - pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); - if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { - tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tdKVStoreEndCommit(pMeta->pStore); - goto _err; - } - } else if (pAct->act == TSDB_DROP_META) { - if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) { - tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tdKVStoreEndCommit(pMeta->pStore); - goto _err; - } + // Skip expired memory data and expired FSET + tsdbSeekCommitIter(&commith, commith.rtn.minKey); + while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { + if (pSet->fid < commith.rtn.minFid) { + tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { - ASSERT(false); + break; } } - if (tdKVStoreEndCommit(pMeta->pStore) < 0) { - tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + // Loop to commit to each file + fid = tsdbNextCommitFid(&(commith)); + while (true) { + // Loop over both on disk and memory + if (pSet == NULL && fid == TSDB_IVLD_FID) break; + + if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { + // Only has existing FSET but no memory data to commit in this + // existing FSET, only check if file in correct retention + if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + pSet = tsdbFSIterNext(&(commith.fsIter)); + } else { + // Has memory data to commit + SDFileSet *pCSet; + int cfid; + + if (pSet == NULL || pSet->fid > fid) { + // Commit to a new FSET with fid: fid + pCSet = NULL; + cfid = fid; + } else { + // Commit to an existing FSET + pCSet = pSet; + cfid = pSet->fid; + pSet = tsdbFSIterNext(&(commith.fsIter)); + } + + if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + fid = tsdbNextCommitFid(&commith); + } } + tsdbDestroyCommitH(&commith); return 0; +} -_err: - return -1; +static void tsdbStartCommit(STsdbRepo *pRepo) { + SMemTable *pMem = pRepo->imem; + + ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); + + tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", + REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); + + tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); + + pRepo->code = TSDB_CODE_SUCCESS; } static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { + if (eno != TSDB_CODE_SUCCESS) { + tsdbEndFSTxnWithError(REPO_FS(pRepo)); + } else { + tsdbEndFSTxn(pRepo); + } + + tsdbInfo("vgId:%d commit over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); - sem_post(&(pRepo->readyToCommit)); + + SMemTable *pIMem = pRepo->imem; + tsdbLockRepo(pRepo); + pRepo->imem = NULL; + tsdbUnlockRepo(pRepo); + tsdbUnRefMemTable(pRepo, pIMem); + tsem_post(&(pRepo->readyToCommit)); } -static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { +#if 0 +static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { for (int i = 0; i < nIters; i++) { TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); - if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1; + if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; + } + return false; +} +#endif + +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + + ASSERT(pSet == NULL || pSet->fid == fid); + + tsdbResetCommitFile(pCommith); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); + + // Set and open files + if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { + return -1; + } + + // Loop to commit each table data + for (int tid = 1; tid < pCommith->niters; tid++) { + SCommitIter *pIter = pCommith->iters + tid; + + if (pIter->pTable == NULL) continue; + + if (tsdbCommitToTable(pCommith, tid) < 0) { + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + } + + if (tsdbWriteBlockIdx(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + // Close commit file + tsdbCloseCommitFile(pCommith, false); + + if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { + return -1; + } + + return 0; +} + +static int tsdbCreateCommitIters(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SMemTable *pMem = pRepo->imem; + STsdbMeta *pMeta = pRepo->tsdbMeta; + + pCommith->niters = pMem->maxTables; + pCommith->iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); + if (pCommith->iters == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (tsdbRLockRepoMeta(pRepo) < 0) return -1; + + // reference all tables + for (int i = 0; i < pMem->maxTables; i++) { + if (pMeta->tables[i] != NULL) { + tsdbRefTable(pMeta->tables[i]); + pCommith->iters[i].pTable = pMeta->tables[i]; + } + } + + if (tsdbUnlockRepoMeta(pRepo) < 0) return -1; + + for (int i = 0; i < pMem->maxTables; i++) { + if ((pCommith->iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && + (TABLE_UID(pCommith->iters[i].pTable) == pMem->tData[i]->uid)) { + if ((pCommith->iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + tSkipListIterNext(pCommith->iters[i].pIter); + } } + return 0; } -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) { - char * dataDir = NULL; - STsdbCfg * pCfg = &pRepo->config; - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pGroup = NULL; - SMemTable * pMem = pRepo->imem; - bool newLast = false; +static void tsdbDestroyCommitIters(SCommitH *pCommith) { + if (pCommith->iters == NULL) return; - TSKEY minKey = 0, maxKey = 0; - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + for (int i = 1; i < pCommith->niters; i++) { + if (pCommith->iters[i].pTable != NULL) { + tsdbUnRefTable(pCommith->iters[i].pTable); + tSkipListDestroyIter(pCommith->iters[i].pIter); + } + } - // Check if there are data to commit to this file - int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey); - if (!hasDataToCommit) { - tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid); - return 0; + free(pCommith->iters); + pCommith->iters = NULL; + pCommith->niters = 0; +} + +// Skip all keys until key (not included) +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); } +} + +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { + STsdbCfg *pCfg = REPO_CFG(pRepo); - // Create and open files for commit - dataDir = tsdbGetDataDirName(pRepo->rootDir); - if (dataDir == NULL) { + memset(pCommith, 0, sizeof(*pCommith)); + tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); + + TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); + + // Init read handle + if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { + return -1; + } + + // Init file iterator + tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); + + if (tsdbCreateCommitIters(pCommith) < 0) { + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pCommith->aBlkIdx == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); return -1; } - if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) { - tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - goto _err; + pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSupBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; } - // Open files for write/read - if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { - tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSubBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; } - newLast = TSDB_NLAST_FILE_OPENED(pHelper); + pCommith->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + if (pCommith->pDataCols == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } - if (tsdbLoadCompIdx(pHelper, NULL) < 0) { - tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + return 0; +} + +static void tsdbDestroyCommitH(SCommitH *pCommith) { + pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); + pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); + pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); + pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); + tsdbDestroyCommitIters(pCommith); + tsdbDestroyReadH(&(pCommith->readh)); + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static int tsdbNextCommitFid(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int fid = TSDB_IVLD_FID; + + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { + continue; + } else { + int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); + if (fid == TSDB_IVLD_FID || fid > tfid) { + fid = tfid; + } + } } - // Loop to commit data in each table - for (int tid = 1; tid < pMem->maxTables; tid++) { - SCommitIter *pIter = iters + tid; - if (pIter->pTable == NULL) continue; + return fid; +} - taosRLockLatch(&(pIter->pTable->latch)); +static int tsdbCommitToTable(SCommitH *pCommith, int tid) { + SCommitIter *pIter = pCommith->iters + tid; + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err; + tsdbResetCommitTable(pCommith); - if (pIter->pIter != NULL) { - if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + TSDB_RLOCK_TABLE(pIter->pTable); + + // Set commit table + if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + // No disk data and no memory data, just return + if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return 0; + } + + // Must has disk data or has memory data + int nBlocks; + int bidx = 0; + SBlock *pBlock; + + if (pCommith->readh.pBlkIdx) { + if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + } else { + nBlocks = 0; + } + + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + + while (true) { + if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; + + if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || + (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { + if (tsdbMoveBlock(pCommith, bidx) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; } - if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) { - taosRUnLockLatch(&(pIter->pTable->latch)); - tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable), - tstrerror(terrno)); - goto _err; + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { + // merge pBlock data and memory data + if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + nextKey = tsdbNextIterKey(pIter->pIter); + } else { + // Only commit memory data + if (pBlock == NULL) { + if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + } else { + if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } } + nextKey = tsdbNextIterKey(pIter->pIter); } + } - taosRUnLockLatch(&(pIter->pTable->latch)); + TSDB_RUNLOCK_TABLE(pIter->pTable); + + if (tsdbWriteBlockInfo(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + return -1; + } - // Move the last block to the new .l file if neccessary - if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { - tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + return 0; +} + +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { + STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + + pCommith->pTable = pTable; + + if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (pCommith->isRFileSet) { + if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { + return -1; + } + } else { + pCommith->readh.pBlkIdx = NULL; + } + return 0; +} + +static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SBlock *pBlock = (SBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } else { + return 0; + } +} + +static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, + bool isSuper) { + STsdbRepo * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlockData *pBlockData; + int64_t offset = 0; + STable * pTable = TSDB_COMMIT_TABLE(pCommith); + int rowsToWrite = pDataCols->numOfRows; + + ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); + ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); + + // Make buffer space + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommith)), TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { + return -1; + } + pBlockData = (SBlockData *)TSDB_COMMIT_BUF(pCommith); + + // Get # of cols not all NULL(not including key column) + int nColsNotAllNull = 0; + for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; + + if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + continue; } - // Write the SCompBlock part - if (tsdbWriteCompInfo(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + memset(pBlockCol, 0, sizeof(*pBlockCol)); + + pBlockCol->colId = pDataCol->colId; + pBlockCol->type = pDataCol->type; + if (tDataTypes[pDataCol->type].statisFunc) { + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), + &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), + &(pBlockCol->numOfNull)); } + nColsNotAllNull++; } - if (tsdbWriteCompIdx(pHelper) < 0) { - tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - goto _err; + ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); + + // Compress the data if neccessary + int tcol = 0; // counter of not all NULL and written columns + uint32_t toffset = 0; + int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); + int32_t lsize = tsize; + int32_t keyLen = 0; + for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + // All not NULL columns finish + if (ncol != 0 && tcol >= nColsNotAllNull) break; + + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + tcol; + + if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; + + int32_t flen; // final length + int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); + void * tptr; + + // Make room + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommith)), lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + return -1; + } + pBlockData = (SBlockData *)TSDB_COMMIT_BUF(pCommith); + pBlockCol = pBlockData->cols + tcol; + tptr = POINTER_SHIFT(pBlockData, lsize); + + if (pCfg->compression == TWO_STAGE_COMP && + tsdbMakeRoom((void **)(&TSDB_COMMIT_COMP_BUF(pCommith)), tlen + COMP_OVERFLOW_BYTES) < 0) { + return -1; + } + + // Compress or just copy + if (pCfg->compression) { + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, + TSDB_COMMIT_COMP_BUF(pCommith), tlen + COMP_OVERFLOW_BYTES); + } else { + flen = tlen; + memcpy(tptr, pDataCol->pData, flen); + } + + // Add checksum + ASSERT(flen > 0); + flen += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); + + if (ncol != 0) { + tsdbSetBlockColOffset(pBlockCol, toffset); + pBlockCol->len = flen; + tcol++; + } else { + keyLen = flen; + } + + toffset += flen; + lsize += flen; + } + + pBlockData->delimiter = TSDB_FILE_DELIMITER; + pBlockData->uid = TABLE_UID(pTable); + pBlockData->numOfCols = nColsNotAllNull; + + taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); + + // Write the whole block to file + if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { + return -1; } - tfree(dataDir); - tsdbCloseHelperFile(pHelper, 0, pGroup); + // Update pBlock membership vairables + pBlock->last = isLast; + pBlock->offset = offset; + pBlock->algorithm = pCfg->compression; + pBlock->numOfRows = rowsToWrite; + pBlock->len = lsize; + pBlock->keyLen = keyLen; + pBlock->numOfSubBlocks = isSuper ? 1 : 0; + pBlock->numOfCols = nColsNotAllNull; + pBlock->keyFirst = dataColsKeyFirst(pDataCols); + pBlock->keyLast = dataColsKeyLast(pDataCols); + + tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, + REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, + pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); + + return 0; +} + +static int tsdbWriteBlockInfo(SCommitH *pCommih) { + SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + SBlockIdx blkIdx; + STable * pTable = TSDB_COMMIT_TABLE(pCommih); + SBlock * pBlock; + size_t nSupBlocks; + size_t nSubBlocks; + uint32_t tlen; + SBlockInfo *pBlkInfo; + int64_t offset; + + nSupBlocks = taosArrayGetSize(pCommih->aSupBlk); + nSubBlocks = taosArrayGetSize(pCommih->aSubBlk); + + if (nSupBlocks <= 0) { + // No data (data all deleted) + return 0; + } + + tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); + + // Write SBlockInfo part + if (tsdbMakeRoom((void **)(&(TSDB_COMMIT_BUF(pCommih))), tlen) < 0) return -1; + pBlkInfo = TSDB_COMMIT_BUF(pCommih); + + pBlkInfo->delimiter = TSDB_FILE_DELIMITER; + pBlkInfo->tid = TABLE_TID(pTable); + pBlkInfo->uid = TABLE_UID(pTable); - pthread_rwlock_wrlock(&(pFileH->fhlock)); + memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pCommih->aSupBlk, 0), nSupBlocks * sizeof(SBlock)); + if (nSubBlocks > 0) { + memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pCommih->aSubBlk, 0), nSubBlocks * sizeof(SBlock)); - (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); - pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; + for (int i = 0; i < nSupBlocks; i++) { + pBlock = pBlkInfo->blocks + i; - if (newLast) { - (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); - pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info; + if (pBlock->numOfSubBlocks > 1) { + pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); + } + } + } + + taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); + + if (tsdbAppendDFile(pHeadf, TSDB_COMMIT_BUF(pCommih), tlen, &offset) < 0) { + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); + + // Set blkIdx + pBlock = taosArrayGet(pCommih->aSupBlk, nSupBlocks - 1); + + blkIdx.tid = TABLE_TID(pTable); + blkIdx.uid = TABLE_UID(pTable); + blkIdx.hasLast = pBlock->last ? 1 : 0; + blkIdx.maxKey = pBlock->keyLast; + blkIdx.numOfBlocks = (uint32_t)nSupBlocks; + blkIdx.len = tlen; + blkIdx.offset = (uint32_t)offset; + + ASSERT(blkIdx.numOfBlocks > 0); + + if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbWriteBlockIdx(SCommitH *pCommih) { + SBlockIdx *pBlkIdx; + SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + size_t nidx = taosArrayGetSize(pCommih->aBlkIdx); + int tlen = 0, size; + int64_t offset; + + if (nidx <= 0) { + // All data are deleted + pHeadf->info.offset = 0; + pHeadf->info.len = 0; + return 0; + } + + for (size_t i = 0; i < nidx; i++) { + pBlkIdx = (SBlockIdx *)taosArrayGet(pCommih->aBlkIdx, i); + + size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommih)), tlen + size) < 0) return -1; + + void *ptr = POINTER_SHIFT(TSDB_COMMIT_BUF(pCommih), tlen); + tsdbEncodeSBlockIdx(&ptr, pBlkIdx); + + tlen += size; + } + + tlen += sizeof(TSCKSUM); + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommih)), tlen) < 0) return -1; + taosCalcChecksumAppend(0, (uint8_t *)TSDB_COMMIT_BUF(pCommih), tlen); + + if (tsdbAppendDFile(pHeadf, TSDB_COMMIT_BUF(pCommih), tlen, &offset) < tlen) { + tsdbError("vgId:%d failed to write block index part to file %s since %s", TSDB_COMMIT_REPO_ID(pCommih), + TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno)); + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(TSDB_COMMIT_BUF(pCommih), tlen - sizeof(TSCKSUM))); + pHeadf->info.offset = (uint32_t)offset; + pHeadf->info.len = tlen; + + return 0; +} + +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SMergeInfo mInfo; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + SDFile * pDFile; + bool isLast; + SBlock block; + + while (true) { + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, + pCfg->update, &mInfo); + + if (pCommith->pDataCols->numOfRows <= 0) break; + + if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } else { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { + return -1; + } + } + + return 0; +} + +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + TSKEY keyLimit; + int16_t colId = 0; + SMergeInfo mInfo; + SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; + SBlock block, supBlock; + SDFile * pDFile; + + if (bidx == nBlocks - 1) { + keyLimit = pCommith->maxKey; } else { - pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info; + keyLimit = pBlock[1].keyFirst - 1; } - pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info; + SSkipListIterator titer = *(pIter->pIter); + if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; + + tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, + pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); + + if (mInfo.nOperations == 0) { + // no new data to insert (all updates denied) + if (tsdbMoveBlock(pCommith, bidx) < 0) { + return -1; + } + *(pIter->pIter) = titer; + } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { + // Ignore the block + ASSERT(0); + *(pIter->pIter) = titer; + } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { + // Add a sub-block + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, + pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, + &mInfo); + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; - pthread_rwlock_unlock(&(pFileH->fhlock)); + if (pBlock->numOfSubBlocks == 1) { + subBlocks[0] = *pBlock; + subBlocks[0].numOfSubBlocks = 0; + } else { + memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + sizeof(SBlock) * pBlock->numOfSubBlocks); + } + subBlocks[pBlock->numOfSubBlocks] = block; + supBlock = *pBlock; + supBlock.keyFirst = mInfo.keyFirst; + supBlock.keyLast = mInfo.keyLast; + supBlock.numOfSubBlocks++; + supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; + supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); + + if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; + } return 0; +} -_err: - tfree(dataDir); - tsdbCloseHelperFile(pHelper, 1, pGroup); - return -1; +static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + SDFile *pDFile; + SBlock block; + bool isSameFile; + + ASSERT(pBlock->numOfSubBlocks > 0); + + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isSameFile = pCommith->isLFileSame; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isSameFile = pCommith->isDFileSame; + } + + if (isSameFile) { + if (pBlock->numOfSubBlocks == 1) { + if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { + return -1; + } + } else { + block = *pBlock; + block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); + + if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + pBlock->numOfSubBlocks) < 0) { + return -1; + } + } + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; } -static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { - SMemTable *pMem = pRepo->imem; - STsdbMeta *pMeta = pRepo->tsdbMeta; +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { + if (taosArrayPush(pCommith->aSupBlk, pSupBlock) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } - SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); - if (iters == NULL) { + if (pSubBlocks && taosArrayPushBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) < 0) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; + return -1; } - if (tsdbRLockRepoMeta(pRepo) < 0) goto _err; + return 0; +} - // reference all tables - for (int i = 0; i < pMem->maxTables; i++) { - if (pMeta->tables[i] != NULL) { - tsdbRefTable(pMeta->tables[i]); - iters[i].pTable = pMeta->tables[i]; +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlock block; + SDFile * pDFile; + bool isLast; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + + int biter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, + pCfg->update); + + if (pCommith->pDataCols->numOfRows == 0) break; + + if (isLastOneBlock) { + if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; } - if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err; + return 0; +} - for (int i = 0; i < pMem->maxTables; i++) { - if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) { - if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update) { + TSKEY key1 = INT64_MAX; + TSKEY key2 = INT64_MAX; + STSchema *pSchema = NULL; + + ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); + tdResetDataCols(pTarget); + + while (true) { + key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + bool isRowDel = false; + SDataRow row = tsdbNextIterRow(pCommitIter->pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } + + if (key1 == INT64_MAX && key2 == INT64_MAX) break; + + if (key1 < key2) { + for (int i = 0; i < pDataCols->numOfCols; i++) { + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); } - tSkipListIterNext(iters[i].pIter); + pTarget->numOfRows++; + (*iter)++; + } else if (key1 > key2) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); + } + + tSkipListIterNext(pCommitIter->pIter); + } else { + if (update) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); + } + } else { + ASSERT(!isRowDel); + + for (int i = 0; i < pDataCols->numOfCols; i++) { + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + pTarget->numOfRows++; + } + (*iter)++; + tSkipListIterNext(pCommitIter->pIter); } + + if (pTarget->numOfRows >= maxRows) break; } +} - return iters; +static void tsdbResetCommitFile(SCommitH *pCommith) { + pCommith->isRFileSet = false; + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + taosArrayClear(pCommith->aBlkIdx); +} -_err: - tsdbDestroyCommitIters(iters, pMem->maxTables); - return NULL; +static void tsdbResetCommitTable(SCommitH *pCommith) { + taosArrayClear(pCommith->aSubBlk); + taosArrayClear(pCommith->aSupBlk); + pCommith->pTable = NULL; } -static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) { - if (iters == NULL) return; +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + SDiskID did; + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); + + tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } - for (int i = 1; i < maxTables; i++) { - if (iters[i].pTable != NULL) { - tsdbUnRefTable(iters[i].pTable); - tSkipListDestroyIter(iters[i].pIter); + // Open read FSET + if (pSet) { + if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { + return -1; } + + pCommith->isRFileSet = true; + + if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + pCommith->isRFileSet = false; } - free(iters); + // Set and open commit FSET + if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { + // Create a new FSET to write data + tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateDFileSet(pWSet, true) < 0) { + tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + return -1; + } + + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); + } else { + did.level = TSDB_FSET_LEVEL(pSet); + did.id = TSDB_FSET_ID(pSet); + + pCommith->wSet.fid = fid; + pCommith->wSet.state = 0; + + // TSDB_FILE_HEAD + SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); + tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); + if (tsdbCreateDFile(pWHeadf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tstrerror(terrno)); + + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + + // TSDB_FILE_DATA + SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); + SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); + tsdbInitDFileEx(pWDataf, pRDataf); + if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + pCommith->isDFileSame = true; + + // TSDB_FILE_LAST + SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); + SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); + if (pRLastf->info.size < 32 * 1024) { + tsdbInitDFileEx(pWLastf, pRLastf); + pCommith->isLFileSame = true; + + if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); + pCommith->isLFileSame = false; + + if (tsdbCreateDFile(pWLastf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } + } + + return 0; } + +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + + if (!hasError) { + TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); + } + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { + if (pBlock->last) { + if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; + } + } + + return false; +} + +static int tsdbApplyRtn(STsdbRepo *pRepo) { + SRtn rtn; + SFSIter fsiter; + STsdbFS * pfs = REPO_FS(pRepo); + SDFileSet *pSet; + + // Get retention snapshot + tsdbGetRtnSnap(pRepo, &rtn); + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + while ((pSet = tsdbFSIterNext(&fsiter))) { + if (pSet->fid < rtn.minFid) { + tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + continue; + } + + if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { + return -1; + } + } + + return 0; +} + +static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { + SDiskID did; + SDFileSet nSet; + STsdbFS * pfs = REPO_FS(pRepo); + int level; + + ASSERT(pSet->fid >= pRtn->minFid); + + level = tsdbGetFidLevel(pSet->fid, pRtn); + + tfsAllocDisk(level, &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + if (did.level > TSDB_FSET_LEVEL(pSet)) { + // Need to move the FSET to higher level + tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); + + if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); + return -1; + } + + if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { + return -1; + } + + tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); + } else { + // On a correct level + if (tsdbUpdateDFileSet(pfs, pSet) < 0) { + return -1; + } + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index c86b8f32b7ff6bfb30e7b734c7b38504200e44e6..9e8e4acd7ebea2209bf08798eb80f300a72927ab 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tlist.h" -#include "tref.h" -#include "tsdbMain.h" +#include "tsdbint.h" typedef struct { bool stop; @@ -34,9 +31,10 @@ typedef struct { static void *tsdbLoopCommit(void *arg); -SCommitQueue tsCommitQueue = {0}; +static SCommitQueue tsCommitQueue = {0}; -int tsdbInitCommitQueue(int nthreads) { +int tsdbInitCommitQueue() { + int nthreads = tsNumOfCommitThreads; SCommitQueue *pQueue = &tsCommitQueue; if (nthreads < 1) nthreads = 1; diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c new file mode 100644 index 0000000000000000000000000000000000000000..cbff4fbeaa6907c83b5836d6039cbfc23a62168b --- /dev/null +++ b/src/tsdb/src/tsdbFS.c @@ -0,0 +1,1204 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tsdbint.h" +#include + +typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; +static const char *tsdbTxnFname[] = {"current.t", "current"}; +#define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) + +static int tsdbComparFidFSet(const void *arg1, const void *arg2); +static void tsdbResetFSStatus(SFSStatus *pStatus); +static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid); +static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo); +static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]); +static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo); +static int tsdbScanAndTryFixFS(STsdbRepo *pRepo); +static int tsdbScanRootDir(STsdbRepo *pRepo); +static int tsdbScanDataDir(STsdbRepo *pRepo); +static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf); +static int tsdbRestoreCurrent(STsdbRepo *pRepo); +static int tsdbComparTFILE(const void *arg1, const void *arg2); + +// ================== CURRENT file header info +static int tsdbEncodeFSHeader(void **buf, SFSHeader *pHeader) { + int tlen = 0; + + tlen += taosEncodeFixedU32(buf, pHeader->version); + tlen += taosEncodeFixedU32(buf, pHeader->len); + + return tlen; +} + +static void *tsdbDecodeFSHeader(void *buf, SFSHeader *pHeader) { + buf = taosDecodeFixedU32(buf, &(pHeader->version)); + buf = taosDecodeFixedU32(buf, &(pHeader->len)); + + return buf; +} + +// ================== STsdbFSMeta +static int tsdbEncodeFSMeta(void **buf, STsdbFSMeta *pMeta) { + int tlen = 0; + + tlen += taosEncodeFixedU32(buf, pMeta->version); + tlen += taosEncodeFixedI64(buf, pMeta->totalPoints); + tlen += taosEncodeFixedI64(buf, pMeta->totalStorage); + + return tlen; +} + +static void *tsdbDecodeFSMeta(void *buf, STsdbFSMeta *pMeta) { + buf = taosDecodeFixedU32(buf, &(pMeta->version)); + buf = taosDecodeFixedI64(buf, &(pMeta->totalPoints)); + buf = taosDecodeFixedI64(buf, &(pMeta->totalStorage)); + + return buf; +} + +// ================== SFSStatus +static int tsdbEncodeDFileSetArray(void **buf, SArray *pArray) { + int tlen = 0; + uint64_t nset = taosArrayGetSize(pArray); + + tlen += taosEncodeFixedU64(buf, nset); + for (size_t i = 0; i < nset; i++) { + SDFileSet *pSet = taosArrayGet(pArray, i); + + tlen += tsdbEncodeDFileSet(buf, pSet); + } + + return tlen; +} + +static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) { + uint64_t nset; + SDFileSet dset; + + taosArrayClear(pArray); + + buf = taosDecodeFixedU64(buf, &nset); + for (size_t i = 0; i < nset; i++) { + buf = tsdbDecodeDFileSet(buf, &dset); + taosArrayPush(pArray, (void *)(&dset)); + } + return buf; +} + +static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) { + ASSERT(pStatus->pmf); + + int tlen = 0; + + tlen += tsdbEncodeSMFile(buf, pStatus->pmf); + tlen += tsdbEncodeDFileSetArray(buf, pStatus->df); + + return tlen; +} + +static void *tsdbDecodeFSStatus(void *buf, SFSStatus *pStatus) { + tsdbResetFSStatus(pStatus); + + pStatus->pmf = &(pStatus->mf); + + buf = tsdbDecodeSMFile(buf, pStatus->pmf); + buf = tsdbDecodeDFileSetArray(buf, pStatus->df); + + return buf; +} + +static SFSStatus *tsdbNewFSStatus(int maxFSet) { + SFSStatus *pStatus = (SFSStatus *)calloc(1, sizeof(*pStatus)); + if (pStatus == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + TSDB_FILE_SET_CLOSED(&(pStatus->mf)); + + pStatus->df = taosArrayInit(maxFSet, sizeof(SDFileSet)); + if (pStatus->df == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + free(pStatus); + return NULL; + } + + return pStatus; +} + +static SFSStatus *tsdbFreeFSStatus(SFSStatus *pStatus) { + if (pStatus) { + pStatus->df = taosArrayDestroy(pStatus->df); + free(pStatus); + } + + return NULL; +} + +static void tsdbResetFSStatus(SFSStatus *pStatus) { + if (pStatus == NULL) { + return; + } + + TSDB_FILE_SET_CLOSED(&(pStatus->mf)); + + pStatus->pmf = NULL; + taosArrayClear(pStatus->df); +} + +static void tsdbSetStatusMFile(SFSStatus *pStatus, const SMFile *pMFile) { + ASSERT(pStatus->pmf == NULL); + + pStatus->pmf = &(pStatus->mf); + tsdbInitMFileEx(pStatus->pmf, (SMFile *)pMFile); +} + +static int tsdbAddDFileSetToStatus(SFSStatus *pStatus, const SDFileSet *pSet) { + if (taosArrayPush(pStatus->df, (void *)pSet) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + TSDB_FSET_SET_CLOSED(((SDFileSet *)taosArrayGetLast(pStatus->df))); + + return 0; +} + +// ================== STsdbFS +STsdbFS *tsdbNewFS(STsdbCfg *pCfg) { + int keep = pCfg->keep; + int days = pCfg->daysPerFile; + int maxFSet = TSDB_MAX_FSETS(keep, days); + STsdbFS *pfs; + + pfs = (STsdbFS *)calloc(1, sizeof(*pfs)); + if (pfs == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + int code = pthread_rwlock_init(&(pfs->lock), NULL); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); + free(pfs); + return NULL; + } + + pfs->cstatus = tsdbNewFSStatus(maxFSet); + if (pfs->cstatus == NULL) { + tsdbFreeFS(pfs); + return NULL; + } + + pfs->metaCache = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pfs->metaCache == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbFreeFS(pfs); + return NULL; + } + + pfs->nstatus = tsdbNewFSStatus(maxFSet); + if (pfs->nstatus == NULL) { + tsdbFreeFS(pfs); + return NULL; + } + + return pfs; +} + +void *tsdbFreeFS(STsdbFS *pfs) { + if (pfs) { + pfs->nstatus = tsdbFreeFSStatus(pfs->nstatus); + taosHashCleanup(pfs->metaCache); + pfs->metaCache = NULL; + pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); + pthread_rwlock_destroy(&(pfs->lock)); + free(pfs); + } + + return NULL; +} + +int tsdbOpenFS(STsdbRepo *pRepo) { + STsdbFS *pfs = REPO_FS(pRepo); + char current[TSDB_FILENAME_LEN] = "\0"; + + ASSERT(pfs != NULL); + + tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + + if (access(current, F_OK) == 0) { + if (tsdbOpenFSFromCurrent(pRepo) < 0) { + tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } else { + if (tsdbRestoreCurrent(pRepo) < 0) { + tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + if (tsdbScanAndTryFixFS(pRepo) < 0) { + tsdbError("vgId:%d failed to scan and fix FS since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // Load meta cache if has meta file + if ((!(pRepo->state & TSDB_STATE_BAD_META)) && tsdbLoadMetaCache(pRepo, true) < 0) { + tsdbError("vgId:%d failed to open FS while loading meta cache since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + return 0; +} + +void tsdbCloseFS(STsdbRepo *pRepo) { + // Do nothing +} + +// Start a new transaction to modify the file system +void tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd) { + STsdbFS *pfs = REPO_FS(pRepo); + ASSERT(pfs->intxn == false); + + pfs->intxn = true; + tsdbResetFSStatus(pfs->nstatus); + pfs->nstatus->meta = pfs->cstatus->meta; + if (pfs->cstatus->pmf == NULL) { + pfs->nstatus->meta.version = 0; + } else { + pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1; + } + pfs->nstatus->meta.totalPoints = pfs->cstatus->meta.totalPoints + pointsAdd; + pfs->nstatus->meta.totalStorage = pfs->cstatus->meta.totalStorage += storageAdd; +} + +void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta) { pfs->nstatus->meta = *pMeta; } + +int tsdbEndFSTxn(STsdbRepo *pRepo) { + STsdbFS *pfs = REPO_FS(pRepo); + ASSERT(FS_IN_TXN(pfs)); + SFSStatus *pStatus; + + // Write current file system snapshot + if (tsdbSaveFSStatus(pfs->nstatus, REPO_ID(pRepo)) < 0) { + tsdbEndFSTxnWithError(pfs); + return -1; + } + + // Make new + tsdbWLockFS(pfs); + pStatus = pfs->cstatus; + pfs->cstatus = pfs->nstatus; + pfs->nstatus = pStatus; + tsdbUnLockFS(pfs); + + // Apply actual change to each file and SDFileSet + tsdbApplyFSTxnOnDisk(pfs->nstatus, pfs->cstatus); + + pfs->intxn = false; + return 0; +} + +int tsdbEndFSTxnWithError(STsdbFS *pfs) { + tsdbApplyFSTxnOnDisk(pfs->nstatus, pfs->cstatus); + // TODO: if mf change, reload pfs->metaCache + pfs->intxn = false; + return 0; +} + +void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile) { tsdbSetStatusMFile(pfs->nstatus, pMFile); } + +int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); } + +static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) { + SFSHeader fsheader; + void * pBuf = NULL; + void * ptr; + char hbuf[TSDB_FILE_HEAD_SIZE] = "\0"; + char tfname[TSDB_FILENAME_LEN] = "\0"; + char cfname[TSDB_FILENAME_LEN] = "\0"; + + tsdbGetTxnFname(vid, TSDB_TXN_TEMP_FILE, tfname); + tsdbGetTxnFname(vid, TSDB_TXN_CURR_FILE, cfname); + + int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + fsheader.version = TSDB_FS_VERSION; + if (pStatus->pmf == NULL) { + ASSERT(taosArrayGetSize(pStatus->df) == 0); + fsheader.len = 0; + } else { + fsheader.len = tsdbEncodeFSStatus(NULL, pStatus) + sizeof(TSCKSUM); + } + + // Encode header part and write + ptr = hbuf; + tsdbEncodeFSHeader(&ptr, &fsheader); + tsdbEncodeFSMeta(&ptr, &(pStatus->meta)); + + taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); + + if (taosWrite(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + terrno = TAOS_SYSTEM_ERROR(errno); + close(fd); + remove(tfname); + return -1; + } + + // Encode file status and write to file + if (fsheader.len > 0) { + if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { + close(fd); + remove(tfname); + return -1; + } + + ptr = pBuf; + tsdbEncodeFSStatus(&ptr, pStatus); + taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); + + if (taosWrite(fd, pBuf, fsheader.len) < fsheader.len) { + terrno = TAOS_SYSTEM_ERROR(errno); + close(fd); + remove(tfname); + taosTZfree(pBuf); + return -1; + } + } + + // fsync, close and rename + if (fsync(fd) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + close(fd); + remove(tfname); + taosTZfree(pBuf); + return -1; + } + + (void)close(fd); + (void)taosRename(tfname, cfname); + taosTZfree(pBuf); + + return 0; +} + +static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo) { + int ifrom = 0; + int ito = 0; + size_t sizeFrom, sizeTo; + SDFileSet *pSetFrom; + SDFileSet *pSetTo; + + sizeFrom = taosArrayGetSize(pFrom->df); + sizeTo = taosArrayGetSize(pTo->df); + + // Apply meta file change + tsdbApplyMFileChange(pFrom->pmf, pTo->pmf); + + // Apply SDFileSet change + if (ifrom >= sizeFrom) { + pSetFrom = NULL; + } else { + pSetFrom = taosArrayGet(pFrom->df, ifrom); + } + + if (ito >= sizeTo) { + pSetTo = NULL; + } else { + pSetTo = taosArrayGet(pTo->df, ito); + } + + while (true) { + if ((pSetTo == NULL) && (pSetFrom == NULL)) break; + + if (pSetTo == NULL || (pSetFrom && pSetFrom->fid < pSetTo->fid)) { + tsdbApplyDFileSetChange(pSetFrom, NULL); + + ifrom++; + if (ifrom >= sizeFrom) { + pSetFrom = NULL; + } else { + pSetFrom = taosArrayGet(pFrom->df, ifrom); + } + } else if (pSetFrom == NULL || pSetFrom->fid > pSetTo->fid) { + // Do nothing + ito++; + if (ito >= sizeTo) { + pSetTo = NULL; + } else { + pSetTo = taosArrayGet(pTo->df, ito); + } + } else { + tsdbApplyDFileSetChange(pSetFrom, pSetTo); + + ifrom++; + if (ifrom >= sizeFrom) { + pSetFrom = NULL; + } else { + pSetFrom = taosArrayGet(pFrom->df, ifrom); + } + + ito++; + if (ito >= sizeTo) { + pSetTo = NULL; + } else { + pSetTo = taosArrayGet(pTo->df, ito); + } + } + } +} + +// ================== SFSIter +// ASSUMPTIONS: the FS Should be read locked when calling these functions +void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction) { + pIter->pfs = pfs; + pIter->direction = direction; + + size_t size = taosArrayGetSize(pfs->cstatus->df); + + pIter->version = pfs->cstatus->meta.version; + + if (size == 0) { + pIter->index = -1; + pIter->fid = TSDB_IVLD_FID; + } else { + if (direction == TSDB_FS_ITER_FORWARD) { + pIter->index = 0; + } else { + pIter->index = (int)(size - 1); + } + + pIter->fid = ((SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index))->fid; + } +} + +void tsdbFSIterSeek(SFSIter *pIter, int fid) { + STsdbFS *pfs = pIter->pfs; + size_t size = taosArrayGetSize(pfs->cstatus->df); + + int flags; + if (pIter->direction == TSDB_FS_ITER_FORWARD) { + flags = TD_GE; + } else { + flags = TD_LE; + } + + void *ptr = taosbsearch(&fid, pfs->cstatus->df->pData, size, sizeof(SDFileSet), tsdbComparFidFSet, flags); + if (ptr == NULL) { + pIter->index = -1; + pIter->fid = TSDB_IVLD_FID; + } else { + pIter->index = (int)(TARRAY_ELEM_IDX(pfs->cstatus->df, ptr)); + pIter->fid = ((SDFileSet *)ptr)->fid; + } +} + +SDFileSet *tsdbFSIterNext(SFSIter *pIter) { + STsdbFS * pfs = pIter->pfs; + SDFileSet *pSet; + + if (pIter->index < 0) { + ASSERT(pIter->fid == TSDB_IVLD_FID); + return NULL; + } + + ASSERT(pIter->fid != TSDB_IVLD_FID); + + if (pIter->version != pfs->cstatus->meta.version) { + pIter->version = pfs->cstatus->meta.version; + tsdbFSIterSeek(pIter, pIter->fid); + } + + if (pIter->index < 0) { + return NULL; + } + + pSet = (SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index); + ASSERT(pSet->fid == pIter->fid); + + if (pIter->direction == TSDB_FS_ITER_FORWARD) { + pIter->index++; + if (pIter->index >= taosArrayGetSize(pfs->cstatus->df)) { + pIter->index = -1; + } + } else { + pIter->index--; + } + + if (pIter->index >= 0) { + pIter->fid = ((SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index))->fid; + } else { + pIter->fid = TSDB_IVLD_FID; + } + + return pSet; +} + +static int tsdbComparFidFSet(const void *arg1, const void *arg2) { + int fid = *(int *)arg1; + SDFileSet *pSet = (SDFileSet *)arg2; + + if (fid < pSet->fid) { + return -1; + } else if (fid == pSet->fid) { + return 0; + } else { + return 1; + } +} + +static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]) { + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", TFS_PRIMARY_PATH(), repoid, tsdbTxnFname[ftype]); +} + +static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + int fd = -1; + void * buffer = NULL; + SFSHeader fsheader; + char current[TSDB_FILENAME_LEN] = "\0"; + void * ptr; + + tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + + // current file exists, try to recover + fd = open(current, O_RDONLY | O_BINARY); + if (fd < 0) { + tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (tsdbMakeRoom(&buffer, TSDB_FILE_HEAD_SIZE) < 0) { + goto _err; + } + + int nread = (int)taosRead(fd, buffer, TSDB_FILE_HEAD_SIZE); + if (nread < 0) { + tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (nread < TSDB_FILE_HEAD_SIZE) { + tsdbError("vgId:%d failed to read header of file %s, read bytes:%d", REPO_ID(pRepo), current, nread); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + if (!taosCheckChecksumWhole((uint8_t *)buffer, TSDB_FILE_HEAD_SIZE)) { + tsdbError("vgId:%d header of file %s failed checksum check", REPO_ID(pRepo), current); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + SFSStatus *pStatus = pfs->cstatus; + ptr = buffer; + ptr = tsdbDecodeFSHeader(ptr, &fsheader); + ptr = tsdbDecodeFSMeta(ptr, &(pStatus->meta)); + + if (fsheader.version != TSDB_FS_VERSION) { + // TODO: handle file version change + } + + if (fsheader.len > 0) { + if (tsdbMakeRoom(&buffer, fsheader.len) < 0) { + goto _err; + } + + nread = (int)taosRead(fd, buffer, fsheader.len); + if (nread < 0) { + tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (nread < fsheader.len) { + tsdbError("vgId:%d failed to read %d bytes from file %s", REPO_ID(pRepo), fsheader.len, current); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + if (!taosCheckChecksumWhole((uint8_t *)buffer, fsheader.len)) { + tsdbError("vgId:%d file %s is corrupted since wrong checksum", REPO_ID(pRepo), current); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + ptr = buffer; + ptr = tsdbDecodeFSStatus(ptr, pStatus); + } else { + tsdbResetFSStatus(pStatus); + } + + taosTZfree(buffer); + close(fd); + + return 0; + +_err: + if (fd >= 0) { + close(fd); + } + taosTZfree(buffer); + return -1; +} + +// Scan and try to fix incorrect files +static int tsdbScanAndTryFixFS(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SFSStatus *pStatus = pfs->cstatus; + + if (tsdbScanAndTryFixMFile(pRepo) < 0) { + tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + size_t size = taosArrayGetSize(pStatus->df); + + for (size_t i = 0; i < size; i++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pStatus->df, i); + + if (tsdbScanAndTryFixDFileSet(pRepo, pSet) < 0) { + tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + // remove those unused files + tsdbScanRootDir(pRepo); + tsdbScanDataDir(pRepo); + return 0; +} + +int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { + char tbuf[128]; + STsdbFS * pfs = REPO_FS(pRepo); + SMFile mf; + SMFile * pMFile = &mf; + void * pBuf = NULL; + SKVRecord rInfo; + int64_t maxBufSize = 0; + SMFInfo minfo; + + taosHashEmpty(pfs->metaCache); + + // No meta file, just return + if (pfs->cstatus->pmf == NULL) return 0; + + mf = pfs->cstatus->mf; + // Load cache first + if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { + return -1; + } + + if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) { + tsdbCloseMFile(pMFile); + return -1; + } + + while (true) { + int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord)); + if (tsize == 0) break; + + if (tsize < 0) { + tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (tsize < sizeof(SKVRecord)) { + tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord), + TSDB_FILE_FULL_NAME(pMFile)); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbCloseMFile(pMFile); + return -1; + } + + void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo); + ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord)); + // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true); + + if (rInfo.offset < 0) { + taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid)); +#if 0 + pStore->info.size += sizeof(SKVRecord); + pStore->info.nRecords--; + pStore->info.nDels++; + pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); +#endif + } else { + ASSERT(rInfo.offset > 0 && rInfo.size > 0); + if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) { + tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pMFile)); + terrno = TSDB_CODE_COM_OUT_OF_MEMORY; + tsdbCloseMFile(pMFile); + return -1; + } + + maxBufSize = MAX(maxBufSize, rInfo.size); + + if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + tsdbCloseMFile(pMFile); + return -1; + } + +#if 0 + pStore->info.size += (sizeof(SKVRecord) + rInfo.size); + pStore->info.nRecords++; +#endif + } + } + + if (recoverMeta) { + pBuf = malloc((size_t)maxBufSize); + if (pBuf == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCloseMFile(pMFile); + return -1; + } + + SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL); + while (pRecord) { + if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { + tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); + if (nread < 0) { + tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + if (nread < pRecord->size) { + tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) { + tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid, + tstrerror(terrno)); + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + pRecord = taosHashIterate(pfs->metaCache, pRecord); + } + + tsdbOrgMeta(pRepo); + } + + tsdbCloseMFile(pMFile); + tfree(pBuf); + return 0; +} + +static int tsdbScanRootDir(STsdbRepo *pRepo) { + char rootDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + STsdbFS * pfs = REPO_FS(pRepo); + const TFILE *pf; + + tsdbGetRootDir(REPO_ID(pRepo), rootDir); + TDIR *tdir = tfsOpendir(rootDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + if (strcmp(bname, tsdbTxnFname[TSDB_TXN_CURR_FILE]) == 0 || strcmp(bname, "data") == 0) { + // Skip current file and data directory + continue; + } + + if (pfs->cstatus->pmf && tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) { + continue; + } + + tfsremove(pf); + tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); + } + + tfsClosedir(tdir); + + return 0; +} + +static int tsdbScanDataDir(STsdbRepo *pRepo) { + char dataDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + STsdbFS * pfs = REPO_FS(pRepo); + const TFILE *pf; + + tsdbGetDataDir(REPO_ID(pRepo), dataDir); + TDIR *tdir = tfsOpendir(dataDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + if (!tsdbIsTFileInFS(pfs, pf)) { + tfsremove(pf); + tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); + } + } + + tfsClosedir(tdir); + + return 0; +} + +static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { + SFSIter fsiter; + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + SDFileSet *pSet; + + while ((pSet = tsdbFSIterNext(&fsiter))) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); + if (tfsIsSameFile(pf, TSDB_FILE_F(pDFile))) { + return true; + } + } + } + + return false; +} + +static int tsdbRestoreMeta(STsdbRepo *pRepo) { + char rootDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + TDIR * tdir = NULL; + const TFILE *pf = NULL; + const char * pattern = "^meta(-ver[0-9]+)?$"; + regex_t regex; + STsdbFS * pfs = REPO_FS(pRepo); + + regcomp(®ex, pattern, REG_EXTENDED); + + tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo)); + + tsdbGetRootDir(REPO_ID(pRepo), rootDir); + + tdir = tfsOpendir(rootDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); + regfree(®ex); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + if (strcmp(bname, "data") == 0) { + // Skip the data/ directory + continue; + } + + if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) { + // Skip current.t file + tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); + tfsremove(pf); + continue; + } + + int code = regexec(®ex, bname, 0, NULL, 0); + if (code == 0) { + // Match + if (pfs->cstatus->pmf != NULL) { + tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), TFILE_NAME(pf)); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tfsClosedir(tdir); + regfree(®ex); + return -1; + } else { + uint32_t version = 0; + if (strcmp(bname, "meta") != 0) { + sscanf(bname, "meta-ver%" PRIu32, &version); + pfs->cstatus->meta.version = version; + } + + pfs->cstatus->pmf = &(pfs->cstatus->mf); + pfs->cstatus->pmf->f = *pf; + TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf); + + if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) { + tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + tfsClosedir(tdir); + regfree(®ex); + return -1; + } + + if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) { + tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbCloseMFile(pfs->cstatus->pmf); + tfsClosedir(tdir); + regfree(®ex); + return -1; + } + + tsdbCloseMFile(pfs->cstatus->pmf); + } + } else if (code == REG_NOMATCH) { + // Not match + tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); + tfsremove(pf); + continue; + } else { + // Has other error + tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + tfsClosedir(tdir); + regfree(®ex); + return -1; + } + } + + if (pfs->cstatus->pmf) { + tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf)); + } else { + tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo)); + } + + tfsClosedir(tdir); + regfree(®ex); + return 0; +} + +static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { + char dataDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + TDIR * tdir = NULL; + const TFILE *pf = NULL; + const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$"; + SArray * fArray = NULL; + regex_t regex; + STsdbFS * pfs = REPO_FS(pRepo); + + tsdbGetDataDir(REPO_ID(pRepo), dataDir); + + // Resource allocation and init + regcomp(®ex, pattern, REG_EXTENDED); + + fArray = taosArrayInit(1024, sizeof(TFILE)); + if (fArray == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, + tstrerror(terrno)); + regfree(®ex); + return -1; + } + + tdir = tfsOpendir(dataDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, + tstrerror(terrno)); + taosArrayDestroy(fArray); + regfree(®ex); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + int code = regexec(®ex, bname, 0, NULL, 0); + if (code == 0) { + if (taosArrayPush(fArray, (void *)pf) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tfsClosedir(tdir); + taosArrayDestroy(fArray); + regfree(®ex); + return -1; + } + } else if (code == REG_NOMATCH) { + // Not match + tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); + tfsremove(pf); + continue; + } else { + // Has other error + tsdbError("vgId:%d failed to restore DFileSet Array while run regexec since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + tfsClosedir(tdir); + taosArrayDestroy(fArray); + regfree(®ex); + return -1; + } + } + + tfsClosedir(tdir); + regfree(®ex); + + // Sort the array according to file name + taosArraySort(fArray, tsdbComparTFILE); + + size_t index = 0; + // Loop to recover each file set + for (;;) { + if (index >= taosArrayGetSize(fArray)) { + break; + } + + SDFileSet fset = {0}; + + TSDB_FSET_SET_CLOSED(&fset); + + // Loop to recover ONE fset + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); + + if (index >= taosArrayGetSize(fArray)) { + tsdbError("vgId:%d incomplete DFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + taosArrayDestroy(fArray); + return -1; + } + + pf = taosArrayGet(fArray, index); + + int tvid, tfid; + TSDB_FILE_T ttype; + uint32_t tversion; + char bname[TSDB_FILENAME_LEN]; + + tfsbasename(pf, bname); + tsdbParseDFilename(bname, &tvid, &tfid, &ttype, &tversion); + + ASSERT(tvid == REPO_ID(pRepo)); + + if (ftype == 0) { + fset.fid = tfid; + } else { + if (tfid != fset.fid) { + tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + taosArrayDestroy(fArray); + return -1; + } + } + + if (ttype != ftype) { + tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + taosArrayDestroy(fArray); + return -1; + } + + pDFile->f = *pf; + + if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); + taosArrayDestroy(fArray); + return -1; + } + + if (tsdbLoadDFileHeader(pDFile, &(pDFile->info)) < 0) { + tsdbError("vgId:%d failed to load DFile %s header since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + tstrerror(terrno)); + taosArrayDestroy(fArray); + return -1; + } + + tsdbCloseDFile(pDFile); + index++; + } + + tsdbInfo("vgId:%d FSET %d is restored", REPO_ID(pRepo), fset.fid); + taosArrayPush(pfs->cstatus->df, &fset); + } + + // Resource release + taosArrayDestroy(fArray); + + return 0; +} + +static int tsdbRestoreCurrent(STsdbRepo *pRepo) { + // Loop to recover mfile + if (tsdbRestoreMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // Loop to recover dfile set + if (tsdbRestoreDFileSet(pRepo) < 0) { + tsdbError("vgId:%d failed to restore DFileSet since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (tsdbSaveFSStatus(pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) { + tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + return 0; +} + +static int tsdbComparTFILE(const void *arg1, const void *arg2) { + TFILE *pf1 = (TFILE *)arg1; + TFILE *pf2 = (TFILE *)arg2; + + int vid1, fid1, vid2, fid2; + TSDB_FILE_T ftype1, ftype2; + uint32_t version1, version2; + char bname1[TSDB_FILENAME_LEN]; + char bname2[TSDB_FILENAME_LEN]; + + tfsbasename(pf1, bname1); + tfsbasename(pf2, bname2); + tsdbParseDFilename(bname1, &vid1, &fid1, &ftype1, &version1); + tsdbParseDFilename(bname2, &vid2, &fid2, &ftype2, &version2); + + if (fid1 < fid2) { + return -1; + } else if (fid1 > fid2) { + return 1; + } else { + if (ftype1 < ftype2) { + return -1; + } else if (ftype1 > ftype2) { + return 1; + } else { + return 0; + } + } +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 5d8933d14142e4c569808fd932efbed63a9ed8e9..8124a0e3b5984eecee1f0934a385be56d5bd3b61 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -12,433 +12,482 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _DEFAULT_SOURCE -#define TAOS_RANDOM_FILE_FAIL_TEST -#include -#include "os.h" -#include "talgo.h" -#include "tchecksum.h" -#include "tsdbMain.h" -#include "tutil.h" +#include "tsdbint.h" -const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"}; +static const char *TSDB_FNAME_SUFFIX[] = { + "head", // TSDB_FILE_HEAD + "data", // TSDB_FILE_DATA + "last", // TSDB_FILE_LAST + "", // TSDB_FILE_MAX + "meta" // TSDB_FILE_META +}; -static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); -static void tsdbDestroyFile(SFile *pFile); -static int compFGroup(const void *arg1, const void *arg2); -static int keyFGroupCompFunc(const void *key, const void *fgroup); -static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo); -static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep); -static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days); +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); +static int tsdbRollBackMFile(SMFile *pMFile); +static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); +static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); +static int tsdbRollBackDFile(SDFile *pDFile); -// ---------------- INTERNAL FUNCTIONS ---------------- -STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { - STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(*pFileH)); - if (pFileH == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } +// ============== SMFile +void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { + char fname[TSDB_FILENAME_LEN]; - int code = pthread_rwlock_init(&(pFileH->fhlock), NULL); - if (code != 0) { - tsdbError("vgId:%d failed to init file handle lock since %s", pCfg->tsdbId, strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK); - pFileH->maxFGroups = TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile); + memset(&(pMFile->info), 0, sizeof(pMFile->info)); + pMFile->info.magic = TSDB_FILE_INIT_MAGIC; - pFileH->pFGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup)); - if (pFileH->pFGroup == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } + tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname); + tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); +} + +void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) { + *pMFile = *pOMFile; + TSDB_FILE_SET_CLOSED(pMFile); +} + +int tsdbEncodeSMFile(void **buf, SMFile *pMFile) { + int tlen = 0; - return pFileH; + tlen += tsdbEncodeMFInfo(buf, &(pMFile->info)); + tlen += tfsEncodeFile(buf, &(pMFile->f)); -_err: - tsdbFreeFileH(pFileH); - return NULL; + return tlen; } -void tsdbFreeFileH(STsdbFileH *pFileH) { - if (pFileH) { - pthread_rwlock_destroy(&pFileH->fhlock); - tfree(pFileH->pFGroup); - free(pFileH); - } +void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) { + buf = tsdbDecodeMFInfo(buf, &(pMFile->info)); + buf = tfsDecodeFile(buf, &(pMFile->f)); + TSDB_FILE_SET_CLOSED(pMFile); + + return buf; +} + +int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) { + int tlen = 0; + + tlen += tsdbEncodeMFInfo(buf, &(pMFile->info)); + tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile)); + + return tlen; } -int tsdbOpenFileH(STsdbRepo *pRepo) { - ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL); +void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) { + char *aname; + buf = tsdbDecodeMFInfo(buf, &(pMFile->info)); + buf = taosDecodeString(buf, &aname); + strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN); + TSDB_FILE_SET_CLOSED(pMFile); + + tfree(aname); - char * tDataDir = NULL; - DIR * dir = NULL; - int fid = 0; - int vid = 0; - regex_t regex1 = {0}, regex2 = {0}; - int code = 0; - char fname[TSDB_FILENAME_LEN] = "\0"; + return buf; +} - SFileGroup fileGroup = {0}; - STsdbFileH *pFileH = pRepo->tsdbFileH; - STsdbCfg * pCfg = &(pRepo->config); +int tsdbApplyMFileChange(SMFile *from, SMFile *to) { + if (from == NULL && to == NULL) return 0; - tDataDir = tsdbGetDataDirName(pRepo->rootDir); - if (tDataDir == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + if (from != NULL) { + if (to == NULL) { + return tsdbRemoveMFile(from); + } else { + if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) { + if (from->info.size > to->info.size) { + tsdbRollBackMFile(to); + } + } else { + return tsdbRemoveMFile(from); + } + } } - dir = opendir(tDataDir); - if (dir == NULL) { - if (errno == ENOENT) { - tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir); - terrno = TAOS_SYSTEM_ERROR(errno); + return 0; +} - if (taosMkDir(tDataDir, 0755) < 0) { - tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; +int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) { + ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC); + + pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pMFile->fd < 0) { + if (errno == ENOENT) { + // Try to create directory recursively + char *s = strdup(TFILE_REL_NAME(&(pMFile->f))); + if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) { + tfree(s); + return -1; } + tfree(s); - dir = opendir(tDataDir); - if (dir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pMFile->fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + return -1; } } else { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + return -1; } } - code = regcomp(®ex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED); - if (code != 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + if (!updateHeader) { + return 0; } - code = regcomp(®ex2, "^v[0-9]+f[0-9]+\\.(h|d|l|s)$", REG_EXTENDED); - if (code != 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + if (tsdbUpdateMFileHeader(pMFile) < 0) { + tsdbCloseMFile(pMFile); + tsdbRemoveMFile(pMFile); + return -1; } - int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); + pMFile->info.size += TSDB_FILE_HEAD_SIZE; - struct dirent *dp = NULL; - while ((dp = readdir(dir)) != NULL) { - if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; + return 0; +} - code = regexec(®ex1, dp->d_name, 0, NULL, 0); - if (code == 0) { - sscanf(dp->d_name, "v%df%d", &vid, &fid); - if (vid != REPO_ID(pRepo)) { - tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name); - continue; - } +int tsdbUpdateMFileHeader(SMFile *pMFile) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - if (fid < mfid) { - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbGetDataFileName(pRepo->rootDir, pCfg->tsdbId, fid, type, fname); - (void)remove(fname); - } - continue; - } + if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) { + return -1; + } - if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue; - memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); - fileGroup.fileId = fid; - - tsdbInitFileGroup(&fileGroup, pRepo); - } else if (code == REG_NOMATCH) { - code = regexec(®ex2, dp->d_name, 0, NULL, 0); - if (code == 0) { - size_t tsize = strlen(tDataDir) + strlen(dp->d_name) + 2; - char * fname1 = malloc(tsize); - if (fname1 == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - sprintf(fname1, "%s/%s", tDataDir, dp->d_name); - - tsize = tsize + 64; - char *fname2 = malloc(tsize); - if (fname2 == NULL) { - free(fname1); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - sprintf(fname2, "%s/%s_back_%" PRId64, tDataDir, dp->d_name, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + void *ptr = buf; + tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile)); - (void)rename(fname1, fname2); + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; + } - tsdbDebug("vgId:%d file %s exists, backup it as %s", REPO_ID(pRepo), fname1, fname2); + return 0; +} - free(fname1); - free(fname2); - continue; - } else if (code == REG_NOMATCH) { - tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name); - continue; - } else { - goto _err; - } - } else { - goto _err; - } +int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - pFileH->pFGroup[pFileH->nFGroups++] = fileGroup; - qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); - tsdbDebug("vgId:%d file group %d is restored, nFGroups %d", REPO_ID(pRepo), fileGroup.fileId, pFileH->nFGroups); - } + ASSERT(TSDB_FILE_OPENED(pMFile)); - regfree(®ex1); - regfree(®ex2); - tfree(tDataDir); - closedir(dir); - return 0; + if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) { + return -1; + } -_err: - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; + } - regfree(®ex1); - regfree(®ex2); + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } - tfree(tDataDir); - if (dir != NULL) closedir(dir); - tsdbCloseFileH(pRepo); - return -1; + tsdbDecodeMFInfo(buf, pInfo); + return 0; } -void tsdbCloseFileH(STsdbRepo *pRepo) { - STsdbFileH *pFileH = pRepo->tsdbFileH; +int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) { + SMFile * pMFile = pRepo->fs->cstatus->pmf; + struct stat mfstat; + SMFile mf; - for (int i = 0; i < pFileH->nFGroups; i++) { - SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbDestroyFile(&pFGroup->files[type]); - } + if (pMFile == NULL) { + // No meta file, no need to scan + return 0; } -} -SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { - STsdbFileH *pFileH = pRepo->tsdbFileH; - STsdbCfg * pCfg = &(pRepo->config); + tsdbInitMFileEx(&mf, pMFile); - if (pFileH->nFGroups >= pFileH->maxFGroups) { - int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); - if (pFileH->pFGroup[0].fileId < mfid) { - pthread_rwlock_wrlock(&pFileH->fhlock); - tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[0])); - pthread_rwlock_unlock(&pFileH->fhlock); - } + if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) { + tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pMFile)); + pRepo->state |= TSDB_STATE_BAD_META; + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); + return 0; } - ASSERT(pFileH->nFGroups < pFileH->maxFGroups); + if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } - SFileGroup fGroup; - SFileGroup *pFGroup = &fGroup; + if (pMFile->info.size < mfstat.st_size) { + if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + return -1; + } - SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); - if (pGroup == NULL) { // if not exists, create one - pFGroup->fileId = fid; - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) { - for (int i = type; i >= 0; i--) { - remove(pFGroup->files[i].fname); - } + if (taosFtruncate(mf.fd, mf.info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseMFile(&mf); + return -1; + } - return NULL; - } + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbCloseMFile(&mf); + return -1; } - pthread_rwlock_wrlock(&pFileH->fhlock); - pFileH->pFGroup[pFileH->nFGroups++] = fGroup; - qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); - pthread_rwlock_unlock(&pFileH->fhlock); - pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); - ASSERT(pGroup != NULL); + tsdbCloseMFile(&mf); + tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + mfstat.st_size, pMFile->info.size); + } else if (pMFile->info.size > mfstat.st_size) { + tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size); + pRepo->state |= TSDB_STATE_BAD_META; + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return 0; + } else { + tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile)); } - return pGroup; + return 0; } -void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { - pIter->pFileH = pFileH; - pIter->direction = direction; +int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) { + int tlen = 0; - if (pFileH->nFGroups == 0) { - pIter->index = -1; - pIter->fileId = -1; - } else { - if (direction == TSDB_FGROUP_ITER_FORWARD) { - pIter->index = 0; - } else { - pIter->index = pFileH->nFGroups - 1; - } - pIter->fileId = pFileH->pFGroup[pIter->index].fileId; - } + tlen += taosEncodeVariantI64(buf, pInfo->size); + tlen += taosEncodeVariantI64(buf, pInfo->tombSize); + tlen += taosEncodeVariantI64(buf, pInfo->nRecords); + tlen += taosEncodeVariantI64(buf, pInfo->nDels); + tlen += taosEncodeFixedU32(buf, pInfo->magic); + + return tlen; +} + +void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) { + buf = taosDecodeVariantI64(buf, &(pInfo->size)); + buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); + buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); + buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); + + return buf; } -void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { - STsdbFileH *pFileH = pIter->pFileH; +static int tsdbRollBackMFile(SMFile *pMFile) { + SMFile mf; - if (pFileH->nFGroups == 0) { - pIter->index = -1; - pIter->fileId = -1; - return; + tsdbInitMFileEx(&mf, pMFile); + + if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + return -1; } - int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; - void *ptr = taosbsearch(&fid, (void *)pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); - if (ptr == NULL) { - pIter->index = -1; - pIter->fileId = -1; - } else { - pIter->index = (int)(POINTER_DISTANCE(ptr, pFileH->pFGroup) / sizeof(SFileGroup)); - pIter->fileId = ((SFileGroup *)ptr)->fileId; + if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseMFile(&mf); + return -1; } + + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbCloseMFile(&mf); + return -1; + } + + TSDB_FILE_FSYNC(&mf); + + tsdbCloseMFile(&mf); + return 0; } -SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { - STsdbFileH *pFileH = pIter->pFileH; - SFileGroup *pFGroup = NULL; +// ============== Operations on SDFile +void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) { + char fname[TSDB_FILENAME_LEN]; - if (pIter->index < 0 || pIter->index >= pFileH->nFGroups || pIter->fileId < 0) return NULL; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK); - pFGroup = &pFileH->pFGroup[pIter->index]; - if (pFGroup->fileId != pIter->fileId) { - tsdbSeekFileGroupIter(pIter, pIter->fileId); - } + TSDB_FILE_SET_CLOSED(pDFile); - if (pIter->index < 0) return NULL; + memset(&(pDFile->info), 0, sizeof(pDFile->info)); + pDFile->info.magic = TSDB_FILE_INIT_MAGIC; - pFGroup = &pFileH->pFGroup[pIter->index]; - ASSERT(pFGroup->fileId == pIter->fileId); + tsdbGetFilename(vid, fid, ver, ftype, fname); + tfsInitFile(&(pDFile->f), did.level, did.id, fname); +} - if (pIter->direction == TSDB_FGROUP_ITER_FORWARD) { - pIter->index++; - } else { - pIter->index--; - } +void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) { + *pDFile = *pODFile; + TSDB_FILE_SET_CLOSED(pDFile); +} - if (pIter->index >= 0 && pIter->index < pFileH->nFGroups) { - pIter->fileId = pFileH->pFGroup[pIter->index].fileId; - } else { - pIter->fileId = -1; - } +int tsdbEncodeSDFile(void **buf, SDFile *pDFile) { + int tlen = 0; - return pFGroup; + tlen += tsdbEncodeDFInfo(buf, &(pDFile->info)); + tlen += tfsEncodeFile(buf, &(pDFile->f)); + + return tlen; } -int tsdbOpenFile(SFile *pFile, int oflag) { - ASSERT(!TSDB_IS_FILE_OPENED(pFile)); +void *tsdbDecodeSDFile(void *buf, SDFile *pDFile) { + buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); + buf = tfsDecodeFile(buf, &(pDFile->f)); + TSDB_FILE_SET_CLOSED(pDFile); - pFile->fd = open(pFile->fname, oflag, 0755); - if (pFile->fd < 0) { - tsdbError("failed to open file %s since %s", pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + return buf; +} - tsdbTrace("open file %s, fd %d", pFile->fname, pFile->fd); +static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) { + int tlen = 0; - return 0; + tlen += tsdbEncodeDFInfo(buf, &(pDFile->info)); + tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pDFile)); + + return tlen; } -void tsdbCloseFile(SFile *pFile) { - if (TSDB_IS_FILE_OPENED(pFile)) { - tsdbTrace("close file %s, fd %d", pFile->fname, pFile->fd); - close(pFile->fd); - pFile->fd = -1; - } +static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { + char *aname; + + buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); + buf = taosDecodeString(buf, &aname); + strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN); + TSDB_FILE_SET_CLOSED(pDFile); + tfree(aname); + + return buf; } -int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { - memset((void *)pFile, 0, sizeof(SFile)); - pFile->fd = -1; +int tsdbCreateDFile(SDFile *pDFile, bool updateHeader) { + ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname); + pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pDFile->fd < 0) { + if (errno == ENOENT) { + // Try to create directory recursively + char *s = strdup(TFILE_REL_NAME(&(pDFile->f))); + if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pDFile), TSDB_FILE_ID(pDFile)) < 0) { + tfree(s); + return -1; + } + tfree(s); - if (access(pFile->fname, F_OK) == 0) { - tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), pFile->fname); - terrno = TSDB_CODE_TDB_FILE_ALREADY_EXISTS; - goto _err; + pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pDFile->fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } } - if (tsdbOpenFile(pFile, O_RDWR | O_CREAT) < 0) { - goto _err; + if (!updateHeader) { + return 0; } - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - - if (tsdbUpdateFileHeader(pFile) < 0) { - tsdbCloseFile(pFile); + if (tsdbUpdateDFileHeader(pDFile) < 0) { + tsdbCloseDFile(pDFile); + tsdbRemoveDFile(pDFile); return -1; } - tsdbCloseFile(pFile); + pDFile->info.size += TSDB_FILE_HEAD_SIZE; return 0; - -_err: - return -1; } -SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) { - void *ptr = - taosbsearch((void *)(&fid), (void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); - if (ptr == NULL) return NULL; - return (SFileGroup *)ptr; +int tsdbUpdateDFileHeader(SDFile *pDFile) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + + if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) { + return -1; + } + + void *ptr = buf; + taosEncodeFixedU32(&ptr, TSDB_FS_VERSION); + tsdbEncodeDFInfo(&ptr, &(pDFile->info)); + + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + if (tsdbWriteDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; + } + + return 0; } -void tsdbFitRetention(STsdbRepo *pRepo) { - STsdbCfg *pCfg = &(pRepo->config); - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pGroup = pFileH->pFGroup; +int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + uint32_t version; - int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); + ASSERT(TSDB_FILE_OPENED(pDFile)); - pthread_rwlock_wrlock(&(pFileH->fhlock)); + if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) { + return -1; + } - while (pFileH->nFGroups > 0 && pGroup[0].fileId < mfid) { - tsdbRemoveFileGroup(pRepo, pGroup); + if (tsdbReadDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; } - pthread_rwlock_unlock(&(pFileH->fhlock)); + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + void *pBuf = buf; + pBuf = taosDecodeFixedU32(pBuf, &version); + pBuf = tsdbDecodeDFInfo(pBuf, pInfo); + return 0; } -int tsdbUpdateFileHeader(SFile *pFile) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; +static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) { + struct stat dfstat; + SDFile df; - void *pBuf = (void *)buf; - taosEncodeFixedU32((void *)(&pBuf), TSDB_FILE_VERSION); - tsdbEncodeSFileInfo((void *)(&pBuf), &(pFile->info)); + tsdbInitDFileEx(&df, pDFile); - taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) { + tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pDFile)); + pRepo->state |= TSDB_STATE_BAD_DATA; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); + return 0; + } - if (lseek(pFile->fd, 0, SEEK_SET) < 0) { - tsdbError("failed to lseek file %s since %s", pFile->fname, strerror(errno)); + if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (taosWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + + if (pDFile->info.size < dfstat.st_size) { + if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + return -1; + } + + if (taosFtruncate(df.fd, df.info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseDFile(&df); + return -1; + } + + if (tsdbUpdateDFileHeader(&df) < 0) { + tsdbCloseDFile(&df); + return -1; + } + + tsdbCloseDFile(&df); + tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + dfstat.st_size, pDFile->info.size); + } else if (pDFile->info.size > dfstat.st_size) { + tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size); + pRepo->state |= TSDB_STATE_BAD_DATA; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return 0; + } else { + tsdbDebug("vgId:%d file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); } return 0; } -int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { +static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { int tlen = 0; + tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); @@ -450,7 +499,7 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { return tlen; } -void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { +static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); @@ -462,156 +511,186 @@ void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { return buf; } -void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { - ASSERT(pFGroup != NULL); - STsdbFileH *pFileH = pRepo->tsdbFileH; - - SFileGroup fileGroup = *pFGroup; +static int tsdbApplyDFileChange(SDFile *from, SDFile *to) { + ASSERT(from != NULL || to != NULL); - int nFilesLeft = pFileH->nFGroups - (int)(POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1); - if (nFilesLeft > 0) { - memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft); - } - - pFileH->nFGroups--; - ASSERT(pFileH->nFGroups >= 0); - - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (remove(fileGroup.files[type].fname) < 0) { - tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); + if (from != NULL) { + if (to == NULL) { + tsdbRemoveDFile(from); + } else { + if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) { + if (from->info.size > to->info.size) { + tsdbRollBackDFile(to); + } + } else { + tsdbRemoveDFile(from); + } } - tsdbDestroyFile(&fileGroup.files[type]); } + + return 0; } -int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; +static int tsdbRollBackDFile(SDFile *pDFile) { + SDFile df = *pDFile; - if (lseek(pFile->fd, 0, SEEK_SET) < 0) { - tsdbError("failed to lseek file %s to start since %s", pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + if (tsdbOpenDFile(&df, O_WRONLY) < 0) { return -1; } - if (taosRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE, - strerror(errno)); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + if (taosFtruncate(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseDFile(&df); return -1; } - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { - tsdbError("file %s header part is corrupted with failed checksum", pFile->fname); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + if (tsdbUpdateDFileHeader(&df) < 0) { + tsdbCloseDFile(&df); return -1; } - void *pBuf = (void *)buf; - pBuf = taosDecodeFixedU32(pBuf, version); - pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + TSDB_FILE_FSYNC(&df); + tsdbCloseDFile(&df); return 0; } -void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) { - uint32_t version = 0; - SFile file; - SFile * pFile = &file; +// ============== Operations on SDFileSet +void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t ver) { + pSet->fid = fid; + pSet->state = 0; - strncpy(pFile->fname, fname, TSDB_FILENAME_LEN - 1); - pFile->fd = -1; - - if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; - if (tsdbLoadFileHeader(pFile, &version) < 0) goto _err; + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); + tsdbInitDFile(pDFile, did, vid, fid, ver, ftype); + } +} - off_t offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) goto _err; - tsdbCloseFile(pFile); +void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype)); + } +} - *magic = pFile->info.magic; - *size = offset; +int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { + int tlen = 0; - return; + tlen += taosEncodeFixedI32(buf, pSet->fid); + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); + } -_err: - tsdbCloseFile(pFile); - *magic = TSDB_FILE_INIT_MAGIC; - *size = 0; + return tlen; } -// ---------------- LOCAL FUNCTIONS ---------------- -static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { - uint32_t version; +void *tsdbDecodeDFileSet(void *buf, SDFileSet *pSet) { + int32_t fid; - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname); + buf = taosDecodeFixedI32(buf, &(fid)); + pSet->state = 0; + pSet->fid = fid; + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + buf = tsdbDecodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); + } + return buf; +} - pFile->fd = -1; - if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; +int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) { + int tlen = 0; - if (tsdbLoadFileHeader(pFile, &version) < 0) { - tsdbError("vgId:%d failed to load file %s header part since %s", REPO_ID(pRepo), pFile->fname, tstrerror(terrno)); - goto _err; + tlen += taosEncodeFixedI32(buf, pSet->fid); + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } - if (pFile->info.size == TSDB_FILE_HEAD_SIZE) { - pFile->info.size = lseek(pFile->fd, 0, SEEK_END); - } + return tlen; +} + +void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) { + int32_t fid; - if (version != TSDB_FILE_VERSION) { - // TODO: deal with error - tsdbError("vgId:%d file %s version %u is not the same as program version %u which may cause problem", - REPO_ID(pRepo), pFile->fname, version, TSDB_FILE_VERSION); + buf = taosDecodeFixedI32(buf, &(fid)); + pSet->fid = fid; + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } + return buf; +} - tsdbCloseFile(pFile); +int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFileFrom = (from) ? TSDB_DFILE_IN_SET(from, ftype) : NULL; + SDFile *pDFileTo = (to) ? TSDB_DFILE_IN_SET(to, ftype) : NULL; + if (tsdbApplyDFileChange(pDFileFrom, pDFileTo) < 0) { + return -1; + } + } return 0; -_err: - tsdbDestroyFile(pFile); - return -1; } -static void tsdbDestroyFile(SFile *pFile) { tsdbCloseFile(pFile); } +int tsdbCreateDFileSet(SDFileSet *pSet, bool updateHeader) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbCreateDFile(TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) { + tsdbCloseDFileSet(pSet); + tsdbRemoveDFileSet(pSet); + return -1; + } + } -static int compFGroup(const void *arg1, const void *arg2) { - int val1 = ((SFileGroup *)arg1)->fileId; - int val2 = ((SFileGroup *)arg2)->fileId; + return 0; +} - if (val1 < val2) { - return -1; - } else if (val1 > val2) { - return 1; - } else { - return 0; +int tsdbUpdateDFileSetHeader(SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbUpdateDFileHeader(TSDB_DFILE_IN_SET(pSet, ftype)) < 0) { + return -1; + } } + return 0; } -static int keyFGroupCompFunc(const void *key, const void *fgroup) { - int fid = *(int *)key; - SFileGroup *pFGroup = (SFileGroup *)fgroup; - if (fid == pFGroup->fileId) { - return 0; - } else { - return fid > pFGroup->fileId ? 1 : -1; +int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) { + return -1; + } } + return 0; } -static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo) { - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbInitFile(&pFGroup->files[type], pRepo, pFGroup->fileId, type) < 0) { - memset(&pFGroup->files[type].info, 0, sizeof(STsdbFileInfo)); - pFGroup->files[type].info.magic = TSDB_FILE_INIT_MAGIC; - pFGroup->state = 1; - pRepo->state = TSDB_STATE_BAD_FILE; - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; +int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version) { + char *p = NULL; + *version = 0; + *ftype = TSDB_FILE_MAX; + + sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, version); + for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) { + if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) { + *ftype = i; + break; } } -} -static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) { - return (TSKEY)(taosGetTimestamp(precision) - keep * tsMsPerDay[precision]); + tfree(p); + return 0; } -static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) { - return (int)(TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision)); +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) { + ASSERT(ftype != TSDB_FILE_MAX); + + if (ftype < TSDB_FILE_MAX) { + if (ver == 0) { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]); + } else { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid, + TSDB_FNAME_SUFFIX[ftype], ver); + } + } else { + if (ver == 0) { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vid, TSDB_FNAME_SUFFIX[ftype]); + } else { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver); + } + } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 3990c0c516d1768135934b098efbcd7746ab424c..8969f61596b33f37cabe934fdd56819b71575315 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -14,130 +14,108 @@ */ // no test file errors here -#include "tsdbMain.h" -#include "os.h" -#include "talgo.h" -#include "taosdef.h" -#include "tchecksum.h" -#include "tscompression.h" -#include "tsdb.h" -#include "tulog.h" - -#define TSDB_CFG_FILE_NAME "config" -#define TSDB_DATA_DIR_NAME "data" -#define TSDB_META_FILE_NAME "meta" -#define TSDB_META_FILE_INDEX 10000000 +#include "tsdbint.h" + #define IS_VALID_PRECISION(precision) \ (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) #define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); -static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg); -static int32_t tsdbUnsetRepoEnv(char *rootDir); -static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg); -static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg); -static char * tsdbGetCfgFname(char *rootDir); -static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg); -static void tsdbFreeRepo(STsdbRepo *pRepo); -static int tsdbRestoreInfo(STsdbRepo *pRepo); -static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); -static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); -static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); -static int keyFGroupCompFunc(const void *key, const void *fgroup); -static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg); -static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg); -static void tsdbStartStream(STsdbRepo *pRepo); -static void tsdbStopStream(STsdbRepo *pRepo); +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); +static void tsdbFreeRepo(STsdbRepo *pRepo); +static void tsdbStartStream(STsdbRepo *pRepo); +static void tsdbStopStream(STsdbRepo *pRepo); // Function declaration -int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { - DIR *dir = opendir(rootDir); - if (dir) { - tsdbDebug("repository %s already exists", rootDir); - closedir(dir); - return 0; - } else { - if (ENOENT != errno) { - tsdbError("failed to open directory %s since %s", rootDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } +int32_t tsdbCreateRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; + char dataDir[TSDB_FILENAME_LEN] = "\0"; - if (mkdir(rootDir, 0755) < 0) { - tsdbError("vgId:%d failed to create rootDir %s since %s", pCfg->tsdbId, rootDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + tsdbGetRootDir(repoid, tsdbDir); + if (tfsMkdir(tsdbDir) < 0) { + goto _err; } - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + tsdbGetDataDir(repoid, dataDir); + if (tfsMkdir(dataDir) < 0) { + goto _err; + } - if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1; + // TODO: need to create current file with nothing in - tsdbDebug( - "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d daysPerFile %d keep " - "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d", - pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, - pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); return 0; + +_err: + tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno)); + return -1; } -int32_t tsdbDropRepo(char *rootDir) { return tsdbUnsetRepoEnv(rootDir); } +int32_t tsdbDropRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; -TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { - STsdbCfg config = {0}; - STsdbRepo *pRepo = NULL; + tsdbGetRootDir(repoid, tsdbDir); + return tfsRmdir(tsdbDir); +} + +STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo; + STsdbCfg config = *pCfg; terrno = TSDB_CODE_SUCCESS; - if (tsdbLoadConfig(rootDir, &config) < 0) { - tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno)); + // Check and set default configurations + if (tsdbCheckAndSetDefaultCfg(&config) < 0) { + tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); return NULL; } - pRepo = tsdbNewRepo(rootDir, pAppH, &config); - if (pRepo == NULL) { - tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno)); + // Create new TSDB object + if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) { + tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId, + tstrerror(terrno)); return NULL; } + // Open meta if (tsdbOpenMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to open meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } if (tsdbOpenBufPool(pRepo) < 0) { - tsdbError("vgId:%d failed to open buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId, + tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } - if (tsdbOpenFileH(pRepo) < 0) { - tsdbError("vgId:%d failed to open file handle since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + if (tsdbOpenFS(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } - if (tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + // TODO: Restore information from data + if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } tsdbStartStream(pRepo); - tsdbDebug("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo)); + tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); - return (TSDB_REPO_T *)pRepo; - -_err: - tsdbCloseRepo(pRepo, false); - return NULL; + return pRepo; } // Note: all working thread and query thread must stopped when calling this function -int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { +int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { if (repo == NULL) return 0; - STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbRepo *pRepo = repo; int vgId = REPO_ID(pRepo); terrno = TSDB_CODE_SUCCESS; @@ -145,16 +123,15 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { tsdbStopStream(pRepo); if (toCommit) { - tsdbAsyncCommit(pRepo); - sem_wait(&(pRepo->readyToCommit)); - terrno = pRepo->code; + tsdbSyncCommit(repo); } + tsdbUnRefMemTable(pRepo, pRepo->mem); tsdbUnRefMemTable(pRepo, pRepo->imem); pRepo->mem = NULL; pRepo->imem = NULL; - tsdbCloseFileH(pRepo); + tsdbCloseFS(pRepo); tsdbCloseBufPool(pRepo); tsdbCloseMeta(pRepo); tsdbFreeRepo(pRepo); @@ -167,88 +144,67 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { } } -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - // STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - uint32_t magic = 0; - char * fname = NULL; - - struct stat fState; +STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { + ASSERT(repo != NULL); + return &((STsdbRepo *)repo)->config; +} - tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); - ASSERT(*index <= eindex); +int tsdbLockRepo(STsdbRepo *pRepo) { + int code = pthread_mutex_lock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pRepo->repoLocked = true; + return 0; +} - char *sdup = strdup(pRepo->rootDir); - char *prefix = dirname(sdup); - int prefixLen = (int)strlen(prefix); +int tsdbUnlockRepo(STsdbRepo *pRepo) { + ASSERT(IS_REPO_LOCKED(pRepo)); + pRepo->repoLocked = false; + int code = pthread_mutex_unlock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} - if (name[0] == 0) { // get the file from index or after, but not larger than eindex - tfree(sdup); - int fid = (*index) / TSDB_FILE_TYPE_MAX; +int tsdbCheckCommit(STsdbRepo *pRepo) { + ASSERT(pRepo->mem != NULL); + STsdbCfg *pCfg = &(pRepo->config); - if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { - if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { - fname = tsdbGetMetaFileName(pRepo->rootDir); - *index = TSDB_META_FILE_INDEX; - magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); - } else { - return 0; - } - } else { - SFileGroup *pFGroup = - taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); - if (pFGroup->fileId == fid) { - fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); - magic = pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].info.magic; - } else { - if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { - fname = strdup(pFGroup->files[0].fname); - *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; - magic = pFGroup->files[0].info.magic; - } else { - return 0; - } - } - } - strcpy(name, fname + prefixLen); - } else { // get the named file at the specified index. If not there, return 0 - fname = malloc(prefixLen + strlen(name) + 2); - sprintf(fname, "%s/%s", prefix, name); - if (access(fname, F_OK) != 0) { - tfree(fname); - tfree(sdup); - return 0; - } - if (*index == TSDB_META_FILE_INDEX) { // get meta file - tsdbGetStoreInfo(fname, &magic, size); - } else { - tsdbGetFileInfoImpl(fname, &magic, size); - } - tfree(fname); - tfree(sdup); - return magic; + STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); + ASSERT(pBufBlock != NULL); + if ((pRepo->mem->extraBuffList != NULL) || + ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { + // trigger commit + if (tsdbAsyncCommit(pRepo) < 0) return -1; } - if (stat(fname, &fState) < 0) { - tfree(fname); - return 0; - } + return 0; +} - *size = fState.st_size; - // magic = *size; +STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; } - tfree(fname); - return magic; -} +STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } + +int tsdbGetState(STsdbRepo *repo) { return repo->state; } -STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { +void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { ASSERT(repo != NULL); - return &((STsdbRepo *)repo)->config; + STsdbRepo *pRepo = repo; + *totalPoints = pRepo->stat.pointsWritten; + *totalStorage = pRepo->stat.totalStorage; + *compStorage = pRepo->stat.compStorage; } -int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { +int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { // TODO: think about multithread cases + return 0; +#if 0 STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbCfg config = pRepo->config; STsdbCfg * pRCfg = &pRepo->config; @@ -281,6 +237,10 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { config.totalBlocks = pCfg->totalBlocks; configChanged = true; } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + config.cacheLastRow = pCfg->cacheLastRow; + configChanged = true; + } if (configChanged) { if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { @@ -290,112 +250,115 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { } return 0; +#endif } -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { - ASSERT(repo != NULL); - STsdbRepo *pRepo = repo; - *totalPoints = pRepo->stat.pointsWritten; - *totalStorage = pRepo->stat.totalStorage; - *compStorage = pRepo->stat.compStorage; -} - -int tsdbGetState(TSDB_REPO_T *repo) { - return ((STsdbRepo *)repo)->state; -} +uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { + // TODO + return 0; +#if 0 + STsdbRepo *pRepo = (STsdbRepo *)repo; + // STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + uint32_t magic = 0; + char * fname = NULL; -// ----------------- INTERNAL FUNCTIONS ----------------- -char *tsdbGetMetaFileName(char *rootDir) { - int tlen = (int)(strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2); - char *fname = calloc(1, tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } + struct stat fState; - snprintf(fname, tlen, "%s/%s", rootDir, TSDB_META_FILE_NAME); - return fname; -} + tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); + ASSERT(*index <= eindex); -void tsdbGetDataFileName(char *rootDir, int vid, int fid, int type, char *fname) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d%s", rootDir, TSDB_DATA_DIR_NAME, vid, fid, tsdbFileSuffix[type]); -} + if (name[0] == 0) { // get the file from index or after, but not larger than eindex + int fid = (*index) / TSDB_FILE_TYPE_MAX; -int tsdbLockRepo(STsdbRepo *pRepo) { - int code = pthread_mutex_lock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; + if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { + if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { + fname = tsdbGetMetaFileName(pRepo->rootDir); + *index = TSDB_META_FILE_INDEX; + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); + sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); + } else { + return 0; + } + } else { + SFileGroup *pFGroup = + taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); + if (pFGroup->fileId == fid) { + SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; + fname = strdup(TSDB_FILE_NAME(pFile)); + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { + SFile *pFile = &pFGroup->files[0]; + fname = strdup(TSDB_FILE_NAME(pFile)); + *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + return 0; + } + } + } + } else { // get the named file at the specified index. If not there, return 0 + fname = malloc(256); + sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); + if (access(fname, F_OK) != 0) { + tfree(fname); + return 0; + } + if (*index == TSDB_META_FILE_INDEX) { // get meta file + tsdbGetStoreInfo(fname, &magic, size); + } else { + char tfname[TSDB_FILENAME_LEN] = "\0"; + sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); + tsdbGetFileInfoImpl(tfname, &magic, size); + } + tfree(fname); + return magic; } - pRepo->repoLocked = true; - return 0; -} -int tsdbUnlockRepo(STsdbRepo *pRepo) { - ASSERT(IS_REPO_LOCKED(pRepo)); - pRepo->repoLocked = false; - int code = pthread_mutex_unlock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; + if (stat(fname, &fState) < 0) { + tfree(fname); + return 0; } - return 0; -} -char *tsdbGetDataDirName(char *rootDir) { - int tlen = (int)(strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2); - char *fname = calloc(1, tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } + *size = fState.st_size; + // magic = *size; - snprintf(fname, tlen, "%s/%s", rootDir, TSDB_DATA_DIR_NAME); - return fname; + tfree(fname); + return magic; +#endif } -int tsdbGetNextMaxTables(int tid) { - ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); - int maxTables = TSDB_INIT_NTABLES; - while (true) { - maxTables = MIN(maxTables, TSDB_MAX_TABLES); - if (tid <= maxTables) break; - maxTables *= 2; - } - - return maxTables + 1; +void tsdbGetRootDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); } -int tsdbCheckCommit(STsdbRepo *pRepo) { - ASSERT(pRepo->mem != NULL); - STsdbCfg *pCfg = &(pRepo->config); - - STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); - ASSERT(pBufBlock != NULL); - if ((pRepo->mem->extraBuffList != NULL) || - ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { - // trigger commit - if (tsdbAsyncCommit(pRepo) < 0) return -1; - } - - return 0; +void tsdbGetDataDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); } -STsdbMeta * tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; } -STsdbFileH * tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; } -STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; } - -// ----------------- LOCAL FUNCTIONS ----------------- static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { + // Check tsdbId + if (pCfg->tsdbId < 0) { + tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + // Check precision if (pCfg->precision == -1) { pCfg->precision = TSDB_DEFAULT_PRECISION; } else { if (!IS_VALID_PRECISION(pCfg->precision)) { tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } @@ -405,16 +368,11 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } else { if (!IS_VALID_COMPRESSION(pCfg->compression)) { tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } - // Check tsdbId - if (pCfg->tsdbId < 0) { - tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); - goto _err; - } - // Check daysPerFile if (pCfg->daysPerFile == -1) { pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; @@ -424,7 +382,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " "%d", pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } @@ -437,7 +396,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " "TSDB_MAX_MIN_ROW_FBLOCK %d", pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } @@ -449,14 +409,16 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " "TSDB_MAX_MAX_ROW_FBLOCK %d", pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } // Check keep @@ -468,417 +430,107 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " "TSDB_MAX_KEEP %d", pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } - // update check - if (pCfg->update != 0) pCfg->update = 1; - - return 0; - -_err: - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; -} - -static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg) { - if (tsdbSaveConfig(rootDir, pCfg) < 0) { - tsdbError("vgId:%d failed to set TSDB environment since %s", pCfg->tsdbId, tstrerror(terrno)); - return -1; - } - - char *dirName = tsdbGetDataDirName(rootDir); - if (dirName == NULL) return -1; - - if (mkdir(dirName, 0755) < 0) { - tsdbError("vgId:%d failed to create directory %s since %s", pCfg->tsdbId, dirName, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - free(dirName); - return -1; - } - - free(dirName); - - char *fname = tsdbGetMetaFileName(rootDir); - if (fname == NULL) return -1; - if (tdCreateKVStore(fname) < 0) { - tsdbError("vgId:%d failed to open KV store since %s", pCfg->tsdbId, tstrerror(terrno)); - free(fname); - return -1; - } - - free(fname); - return 0; -} - -static int32_t tsdbUnsetRepoEnv(char *rootDir) { - taosRemoveDir(rootDir); - tsdbDebug("repository %s is removed", rootDir); - return 0; -} - -static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { - int fd = -1; - char *fname = NULL; - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - char *pBuf = buf; - - fname = tsdbGetCfgFname(rootDir); - if (fname == NULL) { - tsdbError("vgId:%d failed to save configuration since %s", pCfg->tsdbId, tstrerror(terrno)); - goto _err; - } - - fd = open(fname, O_WRONLY | O_CREAT, 0755); - if (fd < 0) { - tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int tlen = tsdbEncodeCfg((void *)(&pBuf), pCfg); - ASSERT((tlen + sizeof(TSCKSUM) <= TSDB_FILE_HEAD_SIZE) && (POINTER_DISTANCE(pBuf, buf) == tlen)); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); - - if (taosWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (fsync(fd) < 0) { - tsdbError("vgId:%d failed to fsync file %s since %s", pCfg->tsdbId, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - free(fname); - close(fd); - return 0; - -_err: - tfree(fname); - if (fd >= 0) close(fd); - return -1; -} - -static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { - char *fname = NULL; - int fd = -1; - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - - fname = tsdbGetCfgFname(rootDir); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - fd = open(fname, O_RDONLY); - if (fd < 0) { - tsdbError("failed to open file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (pCfg->keep1 == 0) { + pCfg->keep1 = pCfg->keep; } - if (taosRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (pCfg->keep2 == 0) { + pCfg->keep2 = pCfg->keep; } - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { - tsdbError("file %s is corrupted", fname); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - goto _err; - } - - tsdbDecodeCfg(buf, pCfg); + // update check + if (pCfg->update != 0) pCfg->update = 1; - tfree(fname); - close(fd); + // update cacheLastRow + if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1; return 0; - -_err: - tfree(fname); - if (fd >= 0) close(fd); - return -1; } -static char *tsdbGetCfgFname(char *rootDir) { - int tlen = (int)(strlen(rootDir) + strlen(TSDB_CFG_FILE_NAME) + 2); - char *fname = calloc(1, tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - snprintf(fname, tlen, "%s/%s", rootDir, TSDB_CFG_FILE_NAME); - return fname; -} - -static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo)); +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); if (pRepo == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + return NULL; } pRepo->state = TSDB_STATE_OK; pRepo->code = TSDB_CODE_SUCCESS; + pRepo->config = *pCfg; + if (pAppH) { + pRepo->appH = *pAppH; + } + pRepo->repoLocked = false; - int code = pthread_mutex_init(&pRepo->mutex, NULL); + int code = pthread_mutex_init(&(pRepo->mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } - code = sem_init(&(pRepo->readyToCommit), 0, 1); + code = tsem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { + code = errno; terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - pRepo->repoLocked = false; - - pRepo->rootDir = strdup(rootDir); - if (pRepo->rootDir == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } - pRepo->config = *pCfg; - if (pAppH) pRepo->appH = *pAppH; - pRepo->tsdbMeta = tsdbNewMeta(pCfg); if (pRepo->tsdbMeta == NULL) { tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } pRepo->pPool = tsdbNewBufPool(pCfg); if (pRepo->pPool == NULL) { tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } - pRepo->tsdbFileH = tsdbNewFileH(pCfg); - if (pRepo->tsdbFileH == NULL) { - tsdbError("vgId:%d failed to create file handle since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + pRepo->fs = tsdbNewFS(pCfg); + if (pRepo->fs == NULL) { + tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; } return pRepo; - -_err: - tsdbFreeRepo(pRepo); - return NULL; } static void tsdbFreeRepo(STsdbRepo *pRepo) { if (pRepo) { - tsdbFreeFileH(pRepo->tsdbFileH); + tsdbFreeFS(pRepo->fs); tsdbFreeBufPool(pRepo->pPool); tsdbFreeMeta(pRepo->tsdbMeta); // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); - tfree(pRepo->rootDir); - sem_destroy(&(pRepo->readyToCommit)); + tsem_destroy(&(pRepo->readyToCommit)); pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } } -static int tsdbRestoreInfo(STsdbRepo *pRepo) { - STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pFGroup = NULL; - - SFileGroupIter iter; - SRWHelper rhelper = {0}; - - if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; - - tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC); - while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { - if (pFGroup->state) continue; - if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; - if (tsdbLoadCompIdx(&rhelper, NULL) < 0) goto _err; - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err; - SCompIdx *pIdx = &(rhelper.curCompIdx); - - if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; - } - } - - tsdbDestroyHelper(&rhelper); - return 0; - -_err: - tsdbDestroyHelper(&rhelper); - return -1; -} - -static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { - int8_t ocompression = pRepo->config.compression; - pRepo->config.compression = compression; - tsdbDebug("vgId:%d tsdb compression is changed from %d to %d", REPO_ID(pRepo), ocompression, compression); -} - -static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { - STsdbCfg * pCfg = &pRepo->config; - STsdbFileH *pFileH = pRepo->tsdbFileH; - int okeep = pCfg->keep; - SFileGroup *pFGroup = NULL; - - ASSERT(pCfg->keep != keep); - int maxFiles = TSDB_MAX_FILE(keep, pCfg->daysPerFile); - - if (maxFiles != pFileH->maxFGroups) { - pthread_rwlock_wrlock(&(pFileH->fhlock)); - - pCfg->keep = keep; - pFGroup = (SFileGroup *)calloc(maxFiles, sizeof(SFileGroup)); - if (pFGroup == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - pthread_rwlock_unlock(&(pFileH->fhlock)); - return -1; - } - - int mfid = (int)(TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - - TSDB_MAX_FILE(keep, pCfg->daysPerFile)); - - int i = 0; - for (; i < pFileH->nFGroups; i++) { - if (pFileH->pFGroup[i].fileId >= mfid) break; - tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[i])); - } - - for (int j = 0; i < pFileH->nFGroups; i++, j++) { - pFGroup[j] = pFileH->pFGroup[i]; - } - - free(pFileH->pFGroup); - pFileH->pFGroup = pFGroup; - - pthread_rwlock_unlock(&(pFileH->fhlock)); - } - - tsdbDebug("vgId:%d keep is changed from %d to %d", REPO_ID(pRepo), okeep, keep); - - return 0; -} - -static int keyFGroupCompFunc(const void *key, const void *fgroup) { - int fid = *(int *)key; - SFileGroup *pFGroup = (SFileGroup *)fgroup; - if (fid == pFGroup->fileId) { - return 0; - } else { - return fid > pFGroup->fileId ? 1 : -1; - } -} - -static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { - int tlen = 0; - - tlen += taosEncodeVariantI32(buf, pCfg->tsdbId); - tlen += taosEncodeFixedI32(buf, pCfg->cacheBlockSize); - tlen += taosEncodeVariantI32(buf, pCfg->totalBlocks); - tlen += taosEncodeVariantI32(buf, pCfg->daysPerFile); - tlen += taosEncodeVariantI32(buf, pCfg->keep); - tlen += taosEncodeVariantI32(buf, pCfg->keep1); - tlen += taosEncodeVariantI32(buf, pCfg->keep2); - tlen += taosEncodeVariantI32(buf, pCfg->minRowsPerFileBlock); - tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); - tlen += taosEncodeFixedI8(buf, pCfg->precision); - tlen += taosEncodeFixedI8(buf, pCfg->compression); - tlen += taosEncodeFixedI8(buf, pCfg->update); - - return tlen; -} - -static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { - buf = taosDecodeVariantI32(buf, &(pCfg->tsdbId)); - buf = taosDecodeFixedI32(buf, &(pCfg->cacheBlockSize)); - buf = taosDecodeVariantI32(buf, &(pCfg->totalBlocks)); - buf = taosDecodeVariantI32(buf, &(pCfg->daysPerFile)); - buf = taosDecodeVariantI32(buf, &(pCfg->keep)); - buf = taosDecodeVariantI32(buf, &(pCfg->keep1)); - buf = taosDecodeVariantI32(buf, &(pCfg->keep2)); - buf = taosDecodeVariantI32(buf, &(pCfg->minRowsPerFileBlock)); - buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); - buf = taosDecodeFixedI8(buf, &(pCfg->precision)); - buf = taosDecodeFixedI8(buf, &(pCfg->compression)); - buf = taosDecodeFixedI8(buf, &(pCfg->update)); - - return buf; -} - -static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { - // TODO - // STsdbCache *pCache = pRepo->tsdbCache; - // int oldNumOfBlocks = pCache->totalCacheBlocks; - - // tsdbLockRepo((TsdbRepoT *)pRepo); - - // ASSERT(pCache->totalCacheBlocks != totalBlocks); - - // if (pCache->totalCacheBlocks < totalBlocks) { - // ASSERT(pCache->totalCacheBlocks == pCache->pool.numOfCacheBlocks); - // int blocksToAdd = pCache->totalCacheBlocks - totalBlocks; - // pCache->totalCacheBlocks = totalBlocks; - // for (int i = 0; i < blocksToAdd; i++) { - // if (tsdbAddCacheBlockToPool(pCache) < 0) { - // tsdbUnLockRepo((TsdbRepoT *)pRepo); - // tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId); - // return -1; - // } - // } - // } else { - // pCache->totalCacheBlocks = totalBlocks; - // tsdbAdjustCacheBlocks(pCache); - // } - // pRepo->config.totalBlocks = totalBlocks; - - // tsdbUnLockRepo((TsdbRepoT *)pRepo); - // tsdbDebug("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, - // totalBlocks); - return 0; -} - -#if 0 - -TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - - STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid); - if (pTable == NULL) return -1; - - return TSDB_GET_TABLE_LAST_KEY(pTable); -} - -#endif - static void tsdbStartStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, tsdbGetTableSchemaImpl(pTable, false, false, -1)); } } } - static void tsdbStopStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -889,3 +541,82 @@ static void tsdbStopStream(STsdbRepo *pRepo) { } } } + +int tsdbRestoreInfo(STsdbRepo *pRepo) { + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlock * pBlock; + + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } + + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + SBlockIdx *pIdx = readh.pBlkIdx; + if (pIdx && lastKey < pIdx->maxKey) { + pTable->lastKey = pIdx->maxKey; + + if (pCfg->cacheLastRow) { + if (tsdbLoadBlockInfo(&readh, NULL) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + pBlock = readh.pBlkInfo->blocks + pIdx->numOfBlocks - 1; + + if (tsdbLoadBlockData(&readh, pBlock, NULL) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + // Get the data in row + ASSERT(pTable->lastRow == NULL); + STSchema *pSchema = tsdbGetTableSchema(pTable); + pTable->lastRow = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); + if (pTable->lastRow == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyReadH(&readh); + return -1; + } + + tdInitDataRow(pTable->lastRow, pSchema); + for (int icol = 0; icol < schemaNCols(pSchema); icol++) { + STColumn *pCol = schemaColAt(pSchema, icol); + SDataCol *pDataCol = readh.pDCols[0]->cols + icol; + tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, + pCol->offset); + } + } + } + + } + } + + tsdbDestroyReadH(&readh); + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 999e2deb413eb13cc7119ee9042a260366c5f9a3..6818f2ed14c278bd5d203d0570f52772c159dcde 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -13,10 +13,22 @@ * along with this program. If not, see . */ -#include "tsdb.h" -#include "tsdbMain.h" +#include "tsdbint.h" #define TSDB_DATA_SKIPLIST_LEVEL 5 +#define TSDB_MAX_INSERT_BATCH 512 + +typedef struct { + int32_t totalLen; + int32_t len; + SDataRow row; +} SSubmitBlkIter; + +typedef struct { + int32_t totalLen; + int32_t len; + void * pMsg; +} SSubmitMsgIter; static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); static void tsdbFreeMemTable(SMemTable *pMemTable); @@ -35,12 +47,13 @@ static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPB static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter); static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter); +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row); static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, TSKEY now); -int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { - STsdbRepo * pRepo = (STsdbRepo *)repo; +int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { + STsdbRepo * pRepo = repo; SSubmitMsgIter msgIter = {0}; SSubmitBlk * pBlock = NULL; int32_t affectedrows = 0; @@ -111,17 +124,66 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { return 0; } -int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { +int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem, SArray *pATable) { + SMemTable *tmem; + + // Get snap object if (tsdbLockRepo(pRepo) < 0) return -1; - *pMem = pRepo->mem; + tmem = pRepo->mem; *pIMem = pRepo->imem; - tsdbRefMemTable(pRepo, *pMem); + tsdbRefMemTable(pRepo, tmem); tsdbRefMemTable(pRepo, *pIMem); if (tsdbUnlockRepo(pRepo) < 0) return -1; - if (*pMem != NULL) taosRLockLatch(&((*pMem)->latch)); + // Copy mem objects and ref needed STableData + if (tmem) { + taosRLockLatch(&(tmem->latch)); + + *pMem = (SMemTable *)calloc(1, sizeof(**pMem)); + if (*pMem == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + taosRUnLockLatch(&(tmem->latch)); + tsdbUnRefMemTable(pRepo, tmem); + tsdbUnRefMemTable(pRepo, *pIMem); + *pMem = NULL; + *pIMem = NULL; + return -1; + } + + (*pMem)->tData = (STableData **)calloc(tmem->maxTables, sizeof(STableData *)); + if ((*pMem)->tData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + taosRUnLockLatch(&(tmem->latch)); + free(*pMem); + tsdbUnRefMemTable(pRepo, tmem); + tsdbUnRefMemTable(pRepo, *pIMem); + *pMem = NULL; + *pIMem = NULL; + return -1; + } + + (*pMem)->keyFirst = tmem->keyFirst; + (*pMem)->keyLast = tmem->keyLast; + (*pMem)->numOfRows = tmem->numOfRows; + (*pMem)->maxTables = tmem->maxTables; + + for (size_t i = 0; i < taosArrayGetSize(pATable); i++) { + STable * pTable = *(STable **)taosArrayGet(pATable, i); + int32_t tid = TABLE_TID(pTable); + STableData *pTableData = (tid < tmem->maxTables) ? tmem->tData[tid] : NULL; + + if ((pTableData == NULL) || (TABLE_UID(pTable) != pTableData->uid)) continue; + + (*pMem)->tData[tid] = tmem->tData[tid]; + T_REF_INC(tmem->tData[tid]); + } + + taosRUnLockLatch(&(tmem->latch)); + } + + tsdbUnRefMemTable(pRepo, tmem); tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); return 0; @@ -131,8 +193,14 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem); if (pMem != NULL) { - taosRUnLockLatch(&(pMem->latch)); - tsdbUnRefMemTable(pRepo, pMem); + for (size_t i = 0; i < pMem->maxTables; i++) { + STableData *pTableData = pMem->tData[i]; + if (pTableData) { + tsdbFreeTableData(pTableData); + } + } + free(pMem->tData); + free(pMem); } if (pIMem != NULL) { @@ -203,11 +271,13 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { } int tsdbAsyncCommit(STsdbRepo *pRepo) { - if (pRepo->mem == NULL) return 0; - - SMemTable *pIMem = pRepo->imem; + tsem_wait(&(pRepo->readyToCommit)); - sem_wait(&(pRepo->readyToCommit)); + ASSERT(pRepo->imem == NULL); + if (pRepo->mem == NULL) { + tsem_post(&(pRepo->readyToCommit)); + return 0; + } if (pRepo->code != TSDB_CODE_SUCCESS) { tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -220,17 +290,15 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { tsdbScheduleCommit(pRepo); if (tsdbUnlockRepo(pRepo) < 0) return -1; - if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1; - return 0; } -int tsdbSyncCommit(TSDB_REPO_T *repo) { - STsdbRepo *pRepo = (STsdbRepo *)repo; +int tsdbSyncCommit(STsdbRepo *repo) { + STsdbRepo *pRepo = repo; tsdbAsyncCommit(pRepo); - sem_wait(&(pRepo->readyToCommit)); - sem_post(&(pRepo->readyToCommit)); + tsem_wait(&(pRepo->readyToCommit)); + tsem_post(&(pRepo->readyToCommit)); if (pRepo->code != TSDB_CODE_SUCCESS) { terrno = pRepo->code; @@ -254,14 +322,17 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) { */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL); + ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; - STSchema *pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; - bool isRowDel = false; - int filterIter = 0; - SDataRow row = NULL; + STSchema * pSchema = NULL; + TSKEY rowKey = 0; + TSKEY fKey = 0; + bool isRowDel = false; + int filterIter = 0; + SDataRow row = NULL; + SMergeInfo mInfo; + + if (pMergeInfo == NULL) pMergeInfo = &mInfo; memset(pMergeInfo, 0, sizeof(*pMergeInfo)); pMergeInfo->keyFirst = INT64_MAX; @@ -420,7 +491,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { STableData *pTableData = (STableData *)calloc(1, sizeof(*pTableData)); if (pTableData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + return NULL; } pTableData->uid = TABLE_UID(pTable); @@ -433,30 +504,27 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + free(pTableData); + return NULL; } - return pTableData; + T_REF_INC(pTableData); -_err: - tsdbFreeTableData(pTableData); - return NULL; + return pTableData; } static void tsdbFreeTableData(STableData *pTableData) { if (pTableData) { - tSkipListDestroy(pTableData->pData); - free(pTableData); + int32_t ref = T_REF_DEC(pTableData); + if (ref == 0) { + tSkipListDestroy(pTableData->pData); + free(pTableData); + } } } static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); } -void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) { - *minKey = fileId * daysPerFile * tsMsPerDay[precision]; - *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; -} - static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { ASSERT(pMemTable->maxTables < maxTables); @@ -606,19 +674,13 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * STable * pTable = NULL; SSubmitBlkIter blkIter = {0}; SDataRow row = NULL; - void ** rows = NULL; + void * rows[TSDB_MAX_INSERT_BATCH] = {0}; int rowCounter = 0; ASSERT(pBlock->tid < pMeta->maxTables); pTable = pMeta->tables[pBlock->tid]; ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); - rows = (void **)calloc(pBlock->numOfRows, sizeof(void *)); - if (rows == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tsdbInitSubmitBlkIter(pBlock, &blkIter); while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) { @@ -632,9 +694,18 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * if (rows[rowCounter] != NULL) { rowCounter++; } + + if (rowCounter == TSDB_MAX_INSERT_BATCH) { + if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + goto _err; + } + + rowCounter = 0; + memset(rows, 0, sizeof(rows)); + } } - if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { goto _err; } @@ -642,11 +713,9 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * pRepo->stat.pointsWritten += points * schemaNCols(pSchema); pRepo->stat.totalStorage += points * schemaVLen(pSchema); - free(rows); return 0; _err: - free(rows); return -1; } @@ -663,9 +732,10 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void return -1; } - if (key > TABLE_LASTKEY(pTable)) { + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + if (key > lastKey) { tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, - REPO_ID(pRepo), key, TABLE_LASTKEY(pTable)); + REPO_ID(pRepo), key, lastKey); return 0; } } @@ -846,8 +916,10 @@ static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **ro if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]); pTableData->numOfRows += dsize; - // TODO: impl delete row thing - if (TABLE_LASTKEY(pTable) < dataRowKey(rows[rowCounter-1])) TABLE_LASTKEY(pTable) = dataRowKey(rows[rowCounter-1]); + // update table latest info + if (tsdbUpdateTableLatestInfo(pRepo, pTable, rows[rowCounter - 1]) < 0) { + return -1; + } return 0; } @@ -889,4 +961,38 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { } } } -} \ No newline at end of file +} + +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { + STsdbCfg *pCfg = &pRepo->config; + + if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { + if (pCfg->cacheLastRow || pTable->lastRow != NULL) { + SDataRow nrow = pTable->lastRow; + if (taosTSizeof(nrow) < dataRowLen(row)) { + SDataRow orow = nrow; + nrow = taosTMalloc(dataRowLen(row)); + if (nrow == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + dataRowCpy(nrow, row); + TSDB_WLOCK_TABLE(pTable); + pTable->lastKey = dataRowKey(row); + pTable->lastRow = nrow; + TSDB_WUNLOCK_TABLE(pTable); + taosTZfree(orow); + } else { + TSDB_WLOCK_TABLE(pTable); + pTable->lastKey = dataRowKey(row); + dataRowCpy(nrow, row); + TSDB_WUNLOCK_TABLE(pTable); + } + } else { + pTable->lastKey = dataRowKey(row); + } + } + + return 0; +} diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 25c815b74e3bd5593bef9157cbd42ba869298cab..5e2e0fce1d45dc8ffceb8c92a2475df5b4da0ad3 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -12,20 +12,12 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include -#include "hash.h" -#include "taosdef.h" -#include "tchecksum.h" -#include "tsdb.h" -#include "tsdbMain.h" -#include "tskiplist.h" +#include "tsdbint.h" #define TSDB_SUPER_TABLE_SL_LEVEL 5 #define DEFAULT_TAG_INDEX_COLUMN 0 static int tsdbCompareSchemaVersion(const void *key1, const void *key2); -static int tsdbRestoreTable(void *pHandle, void *cont, int contLen); -static void tsdbOrgMeta(void *pHandle); static char * getTagIndexKey(const void *pData); static STable *tsdbNewTable(); static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper); @@ -53,12 +45,13 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); // ------------------ OUTER FUNCTIONS ------------------ -int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { +int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; STable * super = NULL; STable * table = NULL; - int newSuper = 0; + bool newSuper = false; + bool superChanged = false; int tid = pCfg->tableId.tid; STable * pTable = NULL; @@ -93,7 +86,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { if (pCfg->type == TSDB_CHILD_TABLE) { super = tsdbGetTableByUid(pMeta, pCfg->superUid); if (super == NULL) { // super table not exists, try to create it - newSuper = 1; + newSuper = true; super = tsdbCreateTableFromCfg(pCfg, true); if (super == NULL) goto _err; } else { @@ -101,6 +94,17 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; goto _err; } + + if (schemaVersion(pCfg->tagSchema) > schemaVersion(super->tagSchema)) { + // tag schema out of date, need to update super table tag version + STSchema *pOldSchema = super->tagSchema; + TSDB_WLOCK_TABLE(super); + super->tagSchema = tdDupSchema(pCfg->tagSchema); + TSDB_WUNLOCK_TABLE(super); + tdFreeSchema(pOldSchema); + + superChanged = true; + } } } @@ -125,7 +129,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { // TODO: refactor duplicate codes int tlen = 0; void *pBuf = NULL; - if (newSuper) { + if (newSuper || superChanged) { tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, super); pBuf = tsdbAllocBytes(pRepo, tlen); if (pBuf == NULL) goto _err; @@ -148,7 +152,7 @@ _err: return -1; } -int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { +int tsdbDropTable(STsdbRepo *repo, STableId tableId) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; uint64_t uid = tableId.uid; @@ -253,7 +257,7 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { } } if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetName(pCfg, pMsg->tableId, true) < 0) goto _err; + if (tsdbTableSetName(pCfg, pMsg->tableFname, true) < 0) goto _err; if (numOfTags > 0) { // Decode tag schema @@ -265,7 +269,7 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { } } if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err; + if (tsdbTableSetSName(pCfg, pMsg->stableFname, true) < 0) goto _err; if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; int32_t tagDataLen = htonl(pMsg->tagDataLen); @@ -301,7 +305,7 @@ static UNUSED_FUNC int32_t colIdCompar(const void* left, const void* right) { return (colId < p2->colId)? -1:1; } -int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { +int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; STSchema * pNewSchema = NULL; @@ -377,11 +381,11 @@ int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { // Chage in memory if (pNewSchema != NULL) { // change super table tag schema - taosWLockLatch(&(pTable->pSuper->latch)); + TSDB_WLOCK_TABLE(pTable->pSuper); STSchema *pOldSchema = pTable->pSuper->tagSchema; pTable->pSuper->tagSchema = pNewSchema; tdFreeSchema(pOldSchema); - taosWUnLockLatch(&(pTable->pSuper->latch)); + TSDB_WUNLOCK_TABLE(pTable->pSuper); } bool isChangeIndexCol = (pMsg->colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))); @@ -392,9 +396,9 @@ int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { tsdbWLockRepoMeta(pRepo); tsdbRemoveTableFromIndex(pMeta, pTable); } - taosWLockLatch(&(pTable->latch)); + TSDB_WLOCK_TABLE(pTable); tdSetKVRowDataOfCol(&(pTable->tagVal), pMsg->colId, pMsg->type, POINTER_SHIFT(pMsg->data, pMsg->schemaLen)); - taosWUnLockLatch(&(pTable->latch)); + TSDB_WUNLOCK_TABLE(pTable); if (isChangeIndexCol) { tsdbAddTableIntoIndex(pMeta, pTable, false); tsdbUnlockRepoMeta(pRepo); @@ -469,6 +473,8 @@ void tsdbFreeMeta(STsdbMeta *pMeta) { } int tsdbOpenMeta(STsdbRepo *pRepo) { + return 0; +#if 0 char * fname = NULL; STsdbMeta *pMeta = pRepo->tsdbMeta; ASSERT(pMeta != NULL); @@ -479,11 +485,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) { goto _err; } - pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo); - if (pMeta->pStore == NULL) { - tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } + // pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo); + // if (pMeta->pStore == NULL) { + // tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno)); + // goto _err; + // } tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); tfree(fname); @@ -492,6 +498,7 @@ int tsdbOpenMeta(STsdbRepo *pRepo) { _err: tfree(fname); return -1; +#endif } int tsdbCloseMeta(STsdbRepo *pRepo) { @@ -500,7 +507,7 @@ int tsdbCloseMeta(STsdbRepo *pRepo) { STable * pTable = NULL; if (pMeta == NULL) return 0; - tdCloseKVStore(pMeta->pStore); + // tdCloseKVStore(pMeta->pStore); for (int i = 1; i < pMeta->maxTables; i++) { tsdbFreeTable(pMeta->tables[i]); } @@ -567,12 +574,13 @@ void tsdbRefTable(STable *pTable) { } void tsdbUnRefTable(STable *pTable) { - int32_t ref = T_REF_DEC(pTable); - tsdbDebug("unref table %s uid:%"PRIu64" tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); + uint64_t uid = TABLE_UID(pTable); + int32_t tid = TABLE_TID(pTable); + int32_t ref = T_REF_DEC(pTable); - if (ref == 0) { - // tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable)); + tsdbDebug("unref table, uid:%" PRIu64 " tid:%d, refCount:%d", uid, tid, ref); + if (ref == 0) { if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { tsdbUnRefTable(pTable->pSuper); } @@ -587,7 +595,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, STable *pCTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; ASSERT(schemaVersion(pSchema) > schemaVersion(pCTable->schema[pCTable->numOfSchemas - 1])); - taosWLockLatch(&(pCTable->latch)); + TSDB_WLOCK_TABLE(pCTable); if (pCTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) { pCTable->schema[pCTable->numOfSchemas++] = pSchema; } else { @@ -599,7 +607,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); - taosWUnLockLatch(&(pCTable->latch)); + TSDB_WUNLOCK_TABLE(pCTable); if (insertAct) { int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pCTable); @@ -609,10 +617,8 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, } } -// ------------------ LOCAL FUNCTIONS ------------------ -static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { - STsdbRepo *pRepo = (STsdbRepo *)pHandle; - STable * pTable = NULL; +int tsdbRestoreTable(STsdbRepo *pRepo, void *cont, int contLen) { + STable *pTable = NULL; if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; @@ -631,8 +637,7 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { return 0; } -static void tsdbOrgMeta(void *pHandle) { - STsdbRepo *pRepo = (STsdbRepo *)pHandle; +void tsdbOrgMeta(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; for (int i = 1; i < pMeta->maxTables; i++) { @@ -643,6 +648,7 @@ static void tsdbOrgMeta(void *pHandle) { } } +// ------------------ LOCAL FUNCTIONS ------------------ static char *getTagIndexKey(const void *pData) { STable *pTable = (STable *)pData; @@ -775,6 +781,7 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); + taosTZfree(pTable->lastRow); tfree(pTable->sql); free(pTable); } @@ -828,7 +835,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo if (lock && tsdbUnlockRepoMeta(pRepo) < 0) return -1; if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, tsdbGetTableSchemaImpl(pTable, false, false, -1)); } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c deleted file mode 100644 index 8e57066d2703b40c647e5f7df9065abe3a77ff20..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbRWHelper.c +++ /dev/null @@ -1,1753 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#define TAOS_RANDOM_FILE_FAIL_TEST -#include "os.h" -#include "talgo.h" -#include "tchecksum.h" -#include "tcoding.h" -#include "tscompression.h" -#include "tsdbMain.h" - -#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) -#define TSDB_KEY_COL_OFFSET 0 -#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock)) -#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last) - -static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); -static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, - bool isLast, bool isSuperBlock); -static int compareKeyBlock(const void *arg1, const void *arg2); -static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize); -static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo); -static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static void tsdbResetHelperFileImpl(SRWHelper *pHelper); -static int tsdbInitHelperFile(SRWHelper *pHelper); -static void tsdbDestroyHelperFile(SRWHelper *pHelper); -static void tsdbResetHelperTableImpl(SRWHelper *pHelper); -static void tsdbResetHelperTable(SRWHelper *pHelper); -static void tsdbInitHelperTable(SRWHelper *pHelper); -static void tsdbDestroyHelperTable(SRWHelper *pHelper); -static void tsdbResetHelperBlockImpl(SRWHelper *pHelper); -static void tsdbResetHelperBlock(SRWHelper *pHelper); -static int tsdbInitHelperBlock(SRWHelper *pHelper); -static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type); -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, - int maxPoints, char *buffer, int bufferSize); -static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, - int numOfColIds); -static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols); -static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx); -static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); -static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey); -static void tsdbDestroyHelperBlock(SRWHelper *pHelper); -static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol, - SDataCol *pDataCol); -static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock); -static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, - int *blkIdx); -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update); -static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps); -static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx); - -// ---------------------- INTERNAL FUNCTIONS ---------------------- -int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { - return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER); -} - -int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { - return tsdbInitHelper(pHelper, pRepo, TSDB_WRITE_HELPER); -} - -void tsdbDestroyHelper(SRWHelper *pHelper) { - if (pHelper) { - taosTZfree(pHelper->pBuffer); - taosTZfree(pHelper->compBuffer); - tsdbDestroyHelperFile(pHelper); - tsdbDestroyHelperTable(pHelper); - tsdbDestroyHelperBlock(pHelper); - memset((void *)pHelper, 0, sizeof(*pHelper)); - } -} - -void tsdbResetHelper(SRWHelper *pHelper) { - if (pHelper) { - // Reset the block part - tsdbResetHelperBlockImpl(pHelper); - - // Reset the table part - tsdbResetHelperTableImpl(pHelper); - - // Reset the file part - tsdbCloseHelperFile(pHelper, false, NULL); - tsdbResetHelperFileImpl(pHelper); - - pHelper->state = TSDB_HELPER_CLEAR_STATE; - } -} - -int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { - ASSERT(pHelper != NULL && pGroup != NULL); - SFile * pFile = NULL; - STsdbRepo *pRepo = pHelper->pRepo; - - // Clear the helper object - tsdbResetHelper(pHelper); - - ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE); - - // Set the files - pHelper->files.fGroup = *pGroup; - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NHEAD, - helperNewHeadF(pHelper)->fname); - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NLAST, - helperNewLastF(pHelper)->fname); - } - - // Open the files - if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) return -1; - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; - if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; - - // Create and open .h - pFile = helperNewHeadF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - if (tsdbUpdateFileHeader(pFile) < 0) return -1; - - // Create and open .l file if should - if (tsdbShouldCreateNewLast(pHelper)) { - pFile = helperNewLastF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - pFile->info.len = 0; - if (tsdbUpdateFileHeader(pFile) < 0) return -1; - } - } else { - if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) return -1; - if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) return -1; - } - - helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN); - - return 0; -} - -int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError, SFileGroup *pGroup) { - SFile *pFile = NULL; - - pFile = helperHeadF(pHelper); - tsdbCloseFile(pFile); - - pFile = helperDataF(pHelper); - if (pFile->fd > 0) { - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - } else { - ASSERT(pGroup != NULL); - taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_DATA].info.size); - } - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - } - - pFile = helperLastF(pHelper); - if (pFile->fd > 0) { - if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - } else { - ASSERT(pGroup != NULL); - taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_LAST].info.size); - } - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - } - - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - pFile = helperNewHeadF(pHelper); - if (pFile->fd > 0) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - if (hasError) (void)remove(pFile->fname); - } - - pFile = helperNewLastF(pHelper); - if (pFile->fd > 0) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - if (hasError) (void)remove(pFile->fname); - } - } - return 0; -} - -int tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { - ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); - - // Clear members and state used by previous table - tsdbResetHelperTable(pHelper); - ASSERT(helperHasState(pHelper, (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD))); - - pHelper->tableInfo.tid = pTable->tableId.tid; - pHelper->tableInfo.uid = pTable->tableId.uid; - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - - if (tdInitDataCols(pHelper->pDataCols[0], pSchema) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (tdInitDataCols(pHelper->pDataCols[1], pSchema) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pHelper->idxH.numOfIdx > 0) { - while (true) { - if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - break; - } - - SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); - if (pIdx->tid == TABLE_TID(pTable)) { - if (pIdx->uid == TABLE_UID(pTable)) { - pHelper->curCompIdx = *pIdx; - } else { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } - pHelper->idxH.curIdx++; - break; - } else if (pIdx->tid > TABLE_TID(pTable)) { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - break; - } else { - pHelper->idxH.curIdx++; - } - } - } else { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } - - if (helperType(pHelper) == TSDB_WRITE_HELPER && pHelper->curCompIdx.hasLast) { - pHelper->hasOldLastBlock = true; - } - - helperSetState(pHelper, TSDB_HELPER_TABLE_SET); - ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1)); - - return 0; -} - -int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - int blkIdx = 0; - - ASSERT(pIdx->offset == 0 || pIdx->uid == TABLE_UID(pCommitIter->pTable)); - if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - - while (true) { - ASSERT(blkIdx <= (int)pIdx->numOfBlocks); - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over - - if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) { - if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1; - blkIdx = pIdx->numOfBlocks; - } else { - if (tsdbProcessMergeCommit(pHelper, pCommitIter, pDataCols, maxKey, &blkIdx) < 0) return -1; - } - } - - return 0; -} - -int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { - STsdbCfg *pCfg = &pHelper->pRepo->config; - - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = &(pHelper->curCompIdx); - SCompBlock compBlock = {0}; - if (TSDB_NLAST_FILE_OPENED(pHelper) && (pHelper->hasOldLastBlock)) { - if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - - SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); - ASSERT(pCompBlock->last); - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); - if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) - return -1; - - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; - -#if 0 - if (pCompBlock->numOfSubBlocks > 1) { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); - if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) - return -1; - - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; - } else { - if (lseek(helperLastF(pHelper)->fd, pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pCompBlock->offset = lseek(helperNewLastF(pHelper)->fd, 0, SEEK_END); - if (pCompBlock->offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewLastF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { - tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo), - helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } -#endif - - pHelper->hasOldLastBlock = false; - } - - return 0; -} - -int tsdbWriteCompInfo(SRWHelper *pHelper) { - SCompIdx *pIdx = &(pHelper->curCompIdx); - off_t offset = 0; - SFile * pFile = helperNewHeadF(pHelper); - - if (pIdx->len > 0) { - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - } else { - pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; - pHelper->pCompInfo->uid = pHelper->tableInfo.uid; - pHelper->pCompInfo->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && - (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - } - - pFile->info.magic = taosCalcChecksum( - pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pIdx->offset = offset; - pIdx->uid = pHelper->tableInfo.uid; - pIdx->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - - if (taosWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosTSizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { - pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, taosTSizeof(pHelper->pWIdx) == 0 ? 1024 : taosTSizeof(pHelper->pWIdx) * 2); - if (pHelper->pWIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - - void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); - pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); - - pFile->info.size += pIdx->len; - // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); - } - - return 0; -} - -int tsdbWriteCompIdx(SRWHelper *pHelper) { - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - off_t offset = 0; - - SFile *pFile = helperNewHeadF(pHelper); - - pFile->info.len += sizeof(TSCKSUM); - if (taosTSizeof(pHelper->pWIdx) < pFile->info.len) { - pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, pFile->info.len); - if (pHelper->pWIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); - pFile->info.magic = taosCalcChecksum( - pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pWIdx, pFile->info.len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - ASSERT(offset == pFile->info.size); - - if (taosWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pFile->info.offset = offset; - pFile->info.size += pFile->info.len; - // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); - - return 0; -} - -int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffer) { - const char *prefixMsg = "failed to load SCompIdx part"; - if (lseek(pFile->fd, offset, SEEK_SET) < 0) { - tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, offset, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosRead(pFile->fd, buffer, len) < len) { - tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len, - strerror(errno)); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)buffer, len)) { - tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, offset, len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - return 0; -} - -int tsdbDecodeSCompIdxImpl(void *buffer, uint32_t len, SCompIdx **ppCompIdx, int *numOfIdx) { - int nIdx = 0; - void *pPtr = buffer; - - while (POINTER_DISTANCE(pPtr, buffer) < (int)(len - sizeof(TSCKSUM))) { - size_t tlen = taosTSizeof(*ppCompIdx); - if (tlen < sizeof(SCompIdx) * (nIdx + 1)) { - *ppCompIdx = (SCompIdx *)taosTRealloc(*ppCompIdx, (tlen == 0) ? 1024 : tlen * 2); - if (*ppCompIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - - pPtr = tsdbDecodeSCompIdx(pPtr, &((*ppCompIdx)[nIdx])); - if (pPtr == NULL) { - tsdbError("failed to decode SCompIdx part, idx:%d", nIdx); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - nIdx++; - - ASSERT(nIdx == 1 || (*ppCompIdx)[nIdx - 1].tid > (*ppCompIdx)[nIdx - 2].tid); - ASSERT(POINTER_DISTANCE(pPtr, buffer) <= (int)(len - sizeof(TSCKSUM))); - } - - *numOfIdx = nIdx; - return 0; -} - -int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { - ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); - SFile *pFile = helperHeadF(pHelper); - - if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { - // If not load from file, just load it in object - if (pFile->info.len > 0) { - if ((pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - // Load SCompIdx binary from file - if (tsdbLoadCompIdxImpl(pFile, pFile->info.offset, pFile->info.len, (void *)(pHelper->pBuffer)) < 0) { - return -1; - } - - // Decode the SCompIdx part - if (tsdbDecodeSCompIdxImpl(pHelper->pBuffer, pFile->info.len, &(pHelper->idxH.pIdxArray), - &(pHelper->idxH.numOfIdx)) < 0) { - tsdbError("vgId:%d failed to decode SCompIdx part from file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - tstrerror(errno)); - return -1; - } - } - } - helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); - - // Copy the memory for outside usage - if (target && pHelper->idxH.numOfIdx > 0) - memcpy(target, pHelper->idxH.pIdxArray, sizeof(SCompIdx) * pHelper->idxH.numOfIdx); - - return 0; -} - -int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) { - const char *prefixMsg = "failed to load SCompInfo/SCompBlock part"; - - if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) { - tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - *ppCompInfo = taosTRealloc((void *)(*ppCompInfo), pIdx->len); - if (*ppCompInfo == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (taosRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) { - tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)(*ppCompInfo), pIdx->len)) { - tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, pIdx->offset, pIdx->len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - return 0; -} - -int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { - ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - - SFile *pFile = helperHeadF(pHelper); - - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (pIdx->offset > 0) { - ASSERT(pIdx->uid == pHelper->tableInfo.uid); - - if (tsdbLoadCompInfoImpl(pFile, pIdx, &(pHelper->pCompInfo)) < 0) return -1; - - ASSERT(pIdx->uid == pHelper->pCompInfo->uid && pIdx->tid == pHelper->pCompInfo->tid); - } - - helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); - } - - if (target) memcpy(target, (void *)(pHelper->pCompInfo), pIdx->len); - - return 0; -} - -int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { - ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - - if (lseek(pFile->fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - size_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); - pHelper->pCompData = taosTRealloc((void *)pHelper->pCompData, tsize); - if (pHelper->pCompData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (taosRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { - tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompData, (uint32_t)tsize)) { - tsdbError("vgId:%d file %s is broken, offset %" PRId64 " size %" PRIzu "", REPO_ID(pHelper->pRepo), pFile->fname, - (int64_t)pCompBlock->offset, tsize); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols); - - if (target) memcpy(target, pHelper->pCompData, tsize); - - return 0; -} - -void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) { - SCompData *pCompData = pHelper->pCompData; - - for (int i = 0, j = 0; i < numOfCols;) { - if (j >= pCompData->numOfCols) { - pStatis[i].numOfNull = -1; - i++; - continue; - } - - if (pStatis[i].colId == pCompData->cols[j].colId) { - pStatis[i].sum = pCompData->cols[j].sum; - pStatis[i].max = pCompData->cols[j].max; - pStatis[i].min = pCompData->cols[j].min; - pStatis[i].maxIndex = pCompData->cols[j].maxIndex; - pStatis[i].minIndex = pCompData->cols[j].minIndex; - pStatis[i].numOfNull = pCompData->cols[j].numOfNull; - i++; - j++; - } else if (pStatis[i].colId < pCompData->cols[j].colId) { - pStatis[i].numOfNull = -1; - i++; - } else { - j++; - } - } -} - -int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo, int16_t *colIds, int numOfColIds) { - ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block - SCompBlock *pTCompBlock = pCompBlock; - - int numOfSubBlocks = pCompBlock->numOfSubBlocks; - if (numOfSubBlocks > 1) - pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); - - tdResetDataCols(pHelper->pDataCols[0]); - if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err; - for (int i = 1; i < numOfSubBlocks; i++) { - tdResetDataCols(pHelper->pDataCols[1]); - pTCompBlock++; - if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[1], colIds, numOfColIds) < 0) goto _err; - if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; - } - - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst && - dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast); - - return 0; - -_err: - return -1; -} - -int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo) { - SCompBlock *pTCompBlock = pCompBlock; - - int numOfSubBlock = pCompBlock->numOfSubBlocks; - if (numOfSubBlock > 1) - pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); - - tdResetDataCols(pHelper->pDataCols[0]); - if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[0]) < 0) goto _err; - for (int i = 1; i < numOfSubBlock; i++) { - tdResetDataCols(pHelper->pDataCols[1]); - pTCompBlock++; - if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[1]) < 0) goto _err; - if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; - } - - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst && - dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast); - - return 0; - -_err: - return -1; -} - -// ---------------------- INTERNAL FUNCTIONS ---------------------- -static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { - ASSERT(helperLastF(pHelper)->fd > 0); - struct stat st; - if (fstat(helperLastF(pHelper)->fd, &st) < 0) return true; - if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true; - return false; -} - -static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, - bool isLast, bool isSuperBlock) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - SCompData *pCompData = (SCompData *)(pHelper->pBuffer); - int64_t offset = 0; - int rowsToWrite = pDataCols->numOfRows; - - ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); - ASSERT(isLast ? rowsToWrite < pCfg->minRowsPerFileBlock : true); - - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int nColsNotAllNull = 0; - for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol *pDataCol = pDataCols->cols + ncol; - SCompCol *pCompCol = pCompData->cols + nColsNotAllNull; - - if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it - continue; - } - - memset(pCompCol, 0, sizeof(*pCompCol)); - - pCompCol->colId = pDataCol->colId; - pCompCol->type = pDataCol->type; - if (tDataTypeDesc[pDataCol->type].getStatisFunc) { - (*tDataTypeDesc[pDataCol->type].getStatisFunc)( - (TSKEY *)(pDataCols->cols[0].pData), pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max), - &(pCompCol->sum), &(pCompCol->minIndex), &(pCompCol->maxIndex), &(pCompCol->numOfNull)); - } - nColsNotAllNull++; - } - - ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); - - // Compress the data if neccessary - int tcol = 0; - int32_t toffset = 0; - int32_t tsize = TSDB_GET_COMPCOL_LEN(nColsNotAllNull); - int32_t lsize = tsize; - int32_t keyLen = 0; - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { - if (ncol != 0 && tcol >= nColsNotAllNull) break; - - SDataCol *pDataCol = pDataCols->cols + ncol; - SCompCol *pCompCol = pCompData->cols + tcol; - - if (ncol != 0 && (pDataCol->colId != pCompCol->colId)) continue; - void *tptr = POINTER_SHIFT(pCompData, lsize); - - int32_t flen = 0; // final length - int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); - - if (pCfg->compression) { - if (pCfg->compression == TWO_STAGE_COMP) { - pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES); - if (pHelper->compBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - - flen = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, - (int32_t)taosTSizeof(pHelper->pBuffer) - lsize, pCfg->compression, - pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer)); - } else { - flen = tlen; - memcpy(tptr, pDataCol->pData, flen); - } - - // Add checksum - ASSERT(flen > 0); - flen += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); - pFile->info.magic = - taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - - if (ncol != 0) { - pCompCol->offset = toffset; - pCompCol->len = flen; - tcol++; - } else { - keyLen = flen; - } - - toffset += flen; - lsize += flen; - } - - pCompData->delimiter = TSDB_FILE_DELIMITER; - pCompData->uid = pHelper->tableInfo.uid; - pCompData->numOfCols = nColsNotAllNull; - - taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize); - pFile->info.magic = taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(pCompData, tsize - sizeof(TSCKSUM)), - sizeof(TSCKSUM)); - - // Write the whole block to file - if (taosWrite(pFile->fd, (void *)pCompData, lsize) < lsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // Update pCompBlock membership vairables - pCompBlock->last = isLast; - pCompBlock->offset = offset; - pCompBlock->algorithm = pCfg->compression; - pCompBlock->numOfRows = rowsToWrite; - pCompBlock->len = lsize; - pCompBlock->keyLen = keyLen; - pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0; - pCompBlock->numOfCols = nColsNotAllNull; - pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); - pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1); - - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 - " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, (int64_t)(pCompBlock->offset), - (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, - pCompBlock->keyLast); - - pFile->info.size += pCompBlock->len; - // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); - - return 0; - -_err: - return -1; -} - -static int compareKeyBlock(const void *arg1, const void *arg2) { - TSKEY key = *(TSKEY *)arg1; - SCompBlock *pBlock = (SCompBlock *)arg2; - - if (key < pBlock->keyFirst) { - return -1; - } else if (key > pBlock->keyLast) { - return 1; - } - - return 0; -} - -static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { - if (taosTSizeof((void *)pHelper->pCompInfo) <= esize) { - size_t tsize = esize + sizeof(SCompBlock) * 16; - pHelper->pCompInfo = (SCompInfo *)taosTRealloc(pHelper->pCompInfo, tsize); - if (pHelper->pCompInfo == NULL) return -1; - } - - return 0; -} - -static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - SCompIdx *pIdx = &(pHelper->curCompIdx); - - ASSERT(blkIdx >= 0 && blkIdx <= (int)pIdx->numOfBlocks); - ASSERT(pCompBlock->numOfSubBlocks == 1); - - // Adjust memory if no more room - if (pIdx->len == 0) pIdx->len = sizeof(SCompInfo) + sizeof(TSCKSUM); - if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err; - - // Change the offset - for (uint32_t i = 0; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); - } - - // Memmove if needed - int tsize = pIdx->len - (sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx); - if (tsize > 0) { - ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) < taosTSizeof(pHelper->pCompInfo)); - ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) + tsize <= taosTSizeof(pHelper->pCompInfo)); - memmove(POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1)), - POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx), tsize); - } - pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; - - pIdx->numOfBlocks++; - pIdx->len += sizeof(SCompBlock); - ASSERT(pIdx->len <= taosTSizeof(pHelper->pCompInfo)); - pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; - pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; - - if (pIdx->numOfBlocks > 1) { - ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); - } - - ASSERT((blkIdx == pIdx->numOfBlocks -1) || (!pCompBlock->last)); - - tsdbDebug("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, - blkIdx); - - return 0; - -_err: - return -1; -} - -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) { - ASSERT(pCompBlock->numOfSubBlocks == 0); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks); - - SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; - ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS); - - size_t spaceNeeded = - (pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock); - if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err; - - pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; - - // Add the sub-block - if (pSCompBlock->numOfSubBlocks > 1) { - size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len)); - if (tsize > 0) { - memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), - (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); - - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); - } - } - - *(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock; - - pSCompBlock->numOfSubBlocks++; - ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); - pSCompBlock->len += sizeof(SCompBlock); - pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; - pSCompBlock->keyFirst = pMergeInfo->keyFirst; - pSCompBlock->keyLast = pMergeInfo->keyLast; - pIdx->len += sizeof(SCompBlock); - } else { // Need to create two sub-blocks - void *ptr = NULL; - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; - if (pTCompBlock->numOfSubBlocks > 1) { - ptr = POINTER_SHIFT(pHelper->pCompInfo, pTCompBlock->offset); - break; - } - } - - if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)); - - size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); - if (tsize > 0) { - memmove(POINTER_SHIFT(ptr, sizeof(SCompBlock) * 2), ptr, tsize); - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2); - } - } - - ((SCompBlock *)ptr)[0] = *pSCompBlock; - ((SCompBlock *)ptr)[0].numOfSubBlocks = 0; - - ((SCompBlock *)ptr)[1] = *pCompBlock; - - pSCompBlock->numOfSubBlocks = 2; - pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; - pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo); - pSCompBlock->len = sizeof(SCompBlock) * 2; - pSCompBlock->keyFirst = pMergeInfo->keyFirst; - pSCompBlock->keyLast = pMergeInfo->keyLast; - - pIdx->len += (sizeof(SCompBlock) * 2); - } - - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; - pIdx->hasLast = (uint32_t)pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; - - tsdbDebug("vgId:%d tid:%d a subblock is added at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); - - return 0; - -_err: - return -1; -} - -static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - ASSERT(pCompBlock->numOfSubBlocks == 1); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - - ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks); - - SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; - - ASSERT(pSCompBlock->numOfSubBlocks >= 1); - - // Delete the sub blocks it has - if (pSCompBlock->numOfSubBlocks > 1) { - size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len)); - if (tsize > 0) { - memmove(POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset), - POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset + pSCompBlock->len), tsize); - } - - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); - } - - pIdx->len -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); - } - - *pSCompBlock = *pCompBlock; - - pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; - pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; - - ASSERT((blkIdx == pIdx->numOfBlocks-1) || (!pCompBlock->last)); - - tsdbDebug("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, - blkIdx); - - return 0; -} - -static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { - SCompIdx *pCompIdx = &(pHelper->curCompIdx); - - ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks); - - SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx); - SCompBlock compBlock = *pCompBlock; - ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); - - if (pCompIdx->numOfBlocks == 1) { - memset(pCompIdx, 0, sizeof(*pCompIdx)); - } else { - int tsize = 0; - - if (compBlock.numOfSubBlocks > 1) { - tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks)); - - ASSERT(tsize > 0); - memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset), - POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks), - tsize); - - pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks; - } - - tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo)); - ASSERT(tsize > 0); - memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize); - - pCompIdx->len -= sizeof(SCompBlock); - - pCompIdx->numOfBlocks--; - pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last); - pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast; - } - - return 0; -} - -static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { - pHelper->idxH.numOfIdx = 0; - pHelper->idxH.curIdx = 0; - memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - helperHeadF(pHelper)->fd = -1; - helperDataF(pHelper)->fd = -1; - helperLastF(pHelper)->fd = -1; - helperNewHeadF(pHelper)->fd = -1; - helperNewLastF(pHelper)->fd = -1; -} - -static int tsdbInitHelperFile(SRWHelper *pHelper) { - tsdbResetHelperFileImpl(pHelper); - return 0; -} - -static void tsdbDestroyHelperFile(SRWHelper *pHelper) { - tsdbCloseHelperFile(pHelper, false, NULL); - tsdbResetHelperFileImpl(pHelper); - taosTZfree(pHelper->idxH.pIdxArray); - taosTZfree(pHelper->pWIdx); -} - -// ---------- Operations on Helper Table part -static void tsdbResetHelperTableImpl(SRWHelper *pHelper) { - memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable)); - pHelper->hasOldLastBlock = false; -} - -static void tsdbResetHelperTable(SRWHelper *pHelper) { - tsdbResetHelperBlock(pHelper); - tsdbResetHelperTableImpl(pHelper); - helperClearState(pHelper, (TSDB_HELPER_TABLE_SET | TSDB_HELPER_INFO_LOAD)); -} - -static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); } - -static void tsdbDestroyHelperTable(SRWHelper *pHelper) { taosTZfree((void *)pHelper->pCompInfo); } - -// ---------- Operations on Helper Block part -static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { - tdResetDataCols(pHelper->pDataCols[0]); - tdResetDataCols(pHelper->pDataCols[1]); -} - -static void tsdbResetHelperBlock(SRWHelper *pHelper) { - tsdbResetHelperBlockImpl(pHelper); - // helperClearState(pHelper, TSDB_HELPER_) -} - -static int tsdbInitHelperBlock(SRWHelper *pHelper) { - STsdbRepo *pRepo = helperRepo(pHelper); - STsdbMeta *pMeta = pHelper->pRepo->tsdbMeta; - - pHelper->pDataCols[0] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); - pHelper->pDataCols[1] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); - if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - tsdbResetHelperBlockImpl(pHelper); - - return 0; -} - -static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { - taosTZfree(pHelper->pCompData); - tdFreeDataCols(pHelper->pDataCols[0]); - tdFreeDataCols(pHelper->pDataCols[1]); -} - -static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { - STsdbCfg *pCfg = &pRepo->config; - memset((void *)pHelper, 0, sizeof(*pHelper)); - STsdbMeta *pMeta = pRepo->tsdbMeta; - - helperType(pHelper) = type; - helperRepo(pHelper) = pRepo; - helperState(pHelper) = TSDB_HELPER_CLEAR_STATE; - - // Init file part - if (tsdbInitHelperFile(pHelper) < 0) goto _err; - - // Init table part - tsdbInitHelperTable(pHelper); - - // Init block part - if (tsdbInitHelperBlock(pHelper) < 0) goto _err; - - // TODO: pMeta->maxRowBytes and pMeta->maxCols may change here causing invalid write - pHelper->pBuffer = - taosTMalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols + - pMeta->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM)); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - return 0; - -_err: - tsdbDestroyHelper(pHelper); - return -1; -} - -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, - int maxPoints, char *buffer, int bufferSize) { - // Verify by checksum - if (!taosCheckChecksumWhole((uint8_t *)content, len)) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - // Decode the data - if (comp) { - // // Need to decompress - int tlen = (*(tDataTypeDesc[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, - pDataCol->spaceSize, comp, buffer, bufferSize); - if (tlen <= 0) { - tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", - len, comp, numOfRows, maxPoints, bufferSize); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - pDataCol->len = tlen; - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - dataColSetOffset(pDataCol, numOfRows); - } - } else { - // No need to decompress, just memcpy it - pDataCol->len = len - sizeof(TSCKSUM); - memcpy(pDataCol->pData, content, pDataCol->len); - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - dataColSetOffset(pDataCol, numOfRows); - } - } - return 0; -} - -static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol, - SDataCol *pDataCol) { - ASSERT(pDataCol->colId == pCompCol->colId); - int tsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; - pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompCol->len); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tsize); - if (pHelper->compBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - int64_t offset = pCompBlock->offset + TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols) + pCompCol->offset; - if (lseek(pFile->fd, (off_t)offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm, - pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock, - pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, - pCompCol->colId, offset); - return -1; - } - - return 0; -} - -static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, int numOfColIds) { - ASSERT(pCompBlock->numOfSubBlocks <= 1); - ASSERT(colIds[0] == 0); - - SFile * pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - SCompCol compCol = {0}; - - // If only load timestamp column, no need to load SCompData part - if (numOfColIds > 1 && tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) goto _err; - - pDataCols->numOfRows = pCompBlock->numOfRows; - - int dcol = 0; - int ccol = 0; - for (int i = 0; i < numOfColIds; i++) { - int16_t colId = colIds[i]; - SDataCol *pDataCol = NULL; - SCompCol *pCompCol = NULL; - - while (true) { - if (dcol >= pDataCols->numOfCols) { - pDataCol = NULL; - break; - } - pDataCol = &pDataCols->cols[dcol]; - if (pDataCol->colId > colId) { - pDataCol = NULL; - break; - } else { - dcol++; - if (pDataCol->colId == colId) break; - } - } - - if (pDataCol == NULL) continue; - ASSERT(pDataCol->colId == colId); - - if (colId == 0) { // load the key row - compCol.colId = colId; - compCol.len = pCompBlock->keyLen; - compCol.type = pDataCol->type; - compCol.offset = TSDB_KEY_COL_OFFSET; - pCompCol = &compCol; - } else { // load non-key rows - while (true) { - if (ccol >= pCompBlock->numOfCols) { - pCompCol = NULL; - break; - } - - pCompCol = &(pHelper->pCompData->cols[ccol]); - if (pCompCol->colId > colId) { - pCompCol = NULL; - break; - } else { - ccol++; - if (pCompCol->colId == colId) break; - } - } - - if (pCompCol == NULL) { - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - continue; - } - - ASSERT(pCompCol->colId == pDataCol->colId); - } - - if (tsdbLoadColData(pHelper, pFile, pCompBlock, pCompCol, pDataCol) < 0) goto _err; - } - - return 0; - -_err: - return -1; -} - -static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { - ASSERT(pCompBlock->numOfSubBlocks <= 1); - - SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - - pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompBlock->len); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - SCompData *pCompData = (SCompData *)pHelper->pBuffer; - - int fd = pFile->fd; - if (lseek(fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (taosRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int32_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); - if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) { - tsdbError("vgId:%d file %s block data is corrupted offset %" PRId64 " len %d", REPO_ID(pHelper->pRepo), - pFile->fname, (int64_t)(pCompBlock->offset), pCompBlock->len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - goto _err; - } - ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); - - pDataCols->numOfRows = pCompBlock->numOfRows; - - // Recover the data - int ccol = 0; // loop iter for SCompCol object - int dcol = 0; // loop iter for SDataCols object - while (dcol < pDataCols->numOfCols) { - SDataCol *pDataCol = &(pDataCols->cols[dcol]); - if (dcol != 0 && ccol >= pCompData->numOfCols) { - // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; - continue; - } - - int16_t tcolId = 0; - int32_t toffset = TSDB_KEY_COL_OFFSET; - int32_t tlen = pCompBlock->keyLen; - - if (dcol != 0) { - SCompCol *pCompCol = &(pCompData->cols[ccol]); - tcolId = pCompCol->colId; - toffset = pCompCol->offset; - tlen = pCompCol->len; - } else { - ASSERT(pDataCol->colId == tcolId); - } - - if (tcolId == pDataCol->colId) { - if (pCompBlock->algorithm == TWO_STAGE_COMP) { - int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows); - } - pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, zsize); - if (pHelper->compBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + toffset, tlen, pCompBlock->algorithm, - pCompBlock->numOfRows, pDataCols->maxPoints, pHelper->compBuffer, - (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d", - REPO_ID(pHelper->pRepo), pFile->fname, tcolId, (int64_t)pCompBlock->offset, toffset); - goto _err; - } - if (dcol != 0) ccol++; - dcol++; - } else if (tcolId < pDataCol->colId) { - ccol++; - } else { - // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; - } - } - - return 0; - -_err: - return -1; -} - -static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) { - int tlen = 0; - - tlen += taosEncodeVariantI32(buf, pIdx->tid); - tlen += taosEncodeVariantU32(buf, pIdx->len); - tlen += taosEncodeVariantU32(buf, pIdx->offset); - tlen += taosEncodeFixedU8(buf, pIdx->hasLast); - tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); - tlen += taosEncodeFixedU64(buf, pIdx->uid); - tlen += taosEncodeFixedU64(buf, pIdx->maxKey); - - return tlen; -} - -static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { - uint8_t hasLast = 0; - uint32_t numOfBlocks = 0; - uint64_t value = 0; - - if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; - if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; - if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; - if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; - pIdx->hasLast = hasLast; - if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; - pIdx->numOfBlocks = numOfBlocks; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->uid = (int64_t)value; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->maxKey = (TSKEY)value; - - return buf; -} - -static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SCompBlock compBlock = {0}; - SMergeInfo mergeInfo = {0}; - SMergeInfo *pMergeInfo = &mergeInfo; - - ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey); - if (pIdx->hasLast) { // append to with last block - ASSERT(pIdx->len > 0); - SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); - ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols, - NULL, 0, pCfg->update, pMergeInfo); - - ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - - if (pDataCols->numOfRows > 0) { - ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); - - if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast); - if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); - - if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); - - if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; - } - - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - } - } else { - ASSERT(!pHelper->hasOldLastBlock); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); - ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - - if (pDataCols->numOfRows > 0) { - ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; - } - } - -#ifndef NDEBUG - TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); - ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey); -#endif - - return 0; -} - -static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, - int *blkIdx) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - SCompBlock compBlock = {0}; - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SDataCols * pDataCols0 = pHelper->pDataCols[0]; - SMergeInfo mergeInfo = {0}; - SMergeInfo *pMergeInfo = &mergeInfo; - SCompBlock oBlock = {0}; - - SSkipListIterator slIter = {0}; - - ASSERT(keyFirst <= pIdx->maxKey); - - SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)blockAtIdx(pHelper, *blkIdx), - pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE); - ASSERT(pCompBlock != NULL); - int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); - oBlock = *pCompBlock; - - ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); - - if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { - while (true) { - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0, - pCfg->update, pMergeInfo); - ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (pDataCols->numOfRows == 0) break; - - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - tblkIdx++; - } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); - } else { - int16_t colId = 0; - if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1; - - TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1); - - slIter = *(pCommitIter->pIter); - tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows, - pCfg->update, pMergeInfo); - - if (pMergeInfo->nOperations == 0) { - // Do nothing - ASSERT(pMergeInfo->rowsDeleteFailed >= 0); - *(pCommitIter->pIter) = slIter; - tblkIdx++; - } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { - // Delete the block and do some stuff - // ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN); - if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1; - *pCommitIter->pIter = slIter; - if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) { - // Append as a sub-block of the searched block - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData, - pDataCols0->numOfRows, pCfg->update, pMergeInfo); - ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0); - if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, - &compBlock, oBlock.last, false) < 0) { - return -1; - } - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) { - return -1; - } - tblkIdx++; - } else { - // load the block data, merge with the memory data - if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock, - pCfg->update); - - if (pDataCols->numOfRows == 0) break; - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - - if (round == 0) { - if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - round++; - tblkIdx++; - } - } - } - - *blkIdx = tblkIdx; - return 0; -} - -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update) { - TSKEY key1 = INT64_MAX; - TSKEY key2 = INT64_MAX; - STSchema *pSchema = NULL; - - ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); - tdResetDataCols(pTarget); - - while (true) { - key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); - bool isRowDel = false; - SDataRow row = tsdbNextIterRow(pCommitIter->pIter); - if (row == NULL || dataRowKey(row) > maxKey) { - key2 = INT64_MAX; - } else { - key2 = dataRowKey(row); - isRowDel = dataRowDeleted(row); - } - - if (key1 == INT64_MAX && key2 == INT64_MAX) break; - - if (key1 < key2) { - for (int i = 0; i < pDataCols->numOfCols; i++) { - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - pTarget->numOfRows++; - (*iter)++; - } else if (key1 > key2) { - if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendDataRowToDataCol(row, pSchema, pTarget); - } - - tSkipListIterNext(pCommitIter->pIter); - } else { - if (update) { - if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendDataRowToDataCol(row, pSchema, pTarget); - } - } else { - ASSERT(!isRowDel); - - for (int i = 0; i < pDataCols->numOfCols; i++) { - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - pTarget->numOfRows++; - } - (*iter)++; - tSkipListIterNext(pCommitIter->pIter); - } - - if (pTarget->numOfRows >= maxRows) break; - } -} - -static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) { - STsdbCfg *pCfg = &(pHelper->pRepo->config); - SFile * pFile = NULL; - bool isLast = false; - - ASSERT(pDataCols->numOfRows > 0); - - if (pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pFile = helperDataF(pHelper); - } else { - isLast = true; - pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? helperNewLastF(pHelper) : helperLastF(pHelper); - } - - ASSERT(pFile->fd > 0); - - if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1; - - return 0; -} - -static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) { - STsdbCfg *pCfg = &(pHelper->pRepo->config); - int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; - - ASSERT(mergeRows > 0); - - if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) { - if (pCompBlock->last) { - if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true; - } else { - if (mergeRows < pCfg->maxRowsPerFileBlock) return true; - } - } - - return false; -} \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d5cc566b5541a41e863d4c136746019b4b172ce0..e0648c33a050fcdb5a6cd83517283092b1061d5e 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -19,10 +19,9 @@ #include "tcompare.h" #include "exception.h" -#include "../../query/inc/qAst.h" // todo move to common module #include "tlosertree.h" -#include "tsdb.h" -#include "tsdbMain.h" +#include "tsdbint.h" +#include "texpr.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -54,7 +53,7 @@ typedef struct SQueryFilePos { } SQueryFilePos; typedef struct SDataBlockLoadInfo { - SFileGroup* fileGroup; + SDFileSet* fileGroup; int32_t slot; int32_t tid; SArray* pLoadedCols; @@ -69,7 +68,7 @@ typedef struct STableCheckInfo { STableId tableId; TSKEY lastKey; STable* pTableObj; - SCompInfo* pCompInfo; + SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks int8_t chosen:2; // indicate which iterator should move forward @@ -79,7 +78,7 @@ typedef struct STableCheckInfo { } STableCheckInfo; typedef struct STableBlockInfo { - SCompBlock* compBlock; + SBlock* compBlock; STableCheckInfo* pTableCheckInfo; } STableBlockInfo; @@ -110,11 +109,13 @@ typedef struct STsdbQueryHandle { SArray* pTableCheckInfo; // SArray int32_t activeIndex; bool checkFiles; // check file stage + bool cachelastrow; // check if last row cached + bool loadExternalRow; // load time window external data rows void* qinfo; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows - SFileGroup* pFileGroup; - SFileGroupIter fileIter; - SRWHelper rhelper; + SDFileSet* pFileGroup; + SFSIter fileIter; + SReadH rhelper; STableBlockInfo* pDataBlockInfo; SDataCols *pDataCols; // in order to hold current file data block @@ -124,6 +125,8 @@ typedef struct STsdbQueryHandle { SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ + SArray *prev; // previous row which is before than time window + SArray *next; // next row which is after the query time window SIOCostSummary cost; } STsdbQueryHandle; @@ -133,15 +136,17 @@ typedef struct STableGroupSupporter { STSchema* pTagSchema; } STableGroupSupporter; -static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); +static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); +static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); +static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle); -static int tsdbCheckInfoCompar(const void* key1, const void* key2); - +static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); +static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); +static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); +static void* doFreeColumnInfoData(SArray* pColumnInfoData); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -182,13 +187,15 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { +static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); SMemRef* pMemRef = pQueryHandle->pMemRef; if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem)); + tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem), psTable); } + + taosArrayDestroy(psTable); } static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { @@ -207,7 +214,37 @@ static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { pQueryHandle->pMemRef = NULL; } -static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) { +int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(pQueryHandle->activeIndex < size && pQueryHandle->activeIndex >= 0 && size >= 1); + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + + + int64_t rows = 0; + SMemRef* pMemRef = pQueryHandle->pMemRef; + if (pMemRef == NULL) { return rows; } + + STableData* pMem = NULL; + STableData* pIMem = NULL; + + SMemTable *pMemT = (SMemTable *)(pMemRef->mem); + SMemTable *pIMemT = (SMemTable *)(pMemRef->imem); + + if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { + pMem = pMemT->tData[pCheckInfo->tableId.tid]; + rows += (pMem && pMem->uid == pCheckInfo->tableId.uid) ? pMem->numOfRows: 0; + } + if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { + pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; + rows += (pIMem && pIMem->uid == pCheckInfo->tableId.uid) ? pIMem->numOfRows: 0; + } + + return rows; +} + +static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); assert(sizeOfGroup >= 1 && pMeta != NULL); @@ -217,6 +254,12 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa return NULL; } + SArray* pTable = taosArrayInit(4, sizeof(STable*)); + if (pTable == NULL) { + taosArrayDestroy(pTableCheckInfo); + return NULL; + } + // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(pGroupList->pGroupList, i); @@ -249,30 +292,46 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa } taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); + + size_t gsize = taosArrayGetSize(pTableCheckInfo); + + for (int32_t i = 0; i < gsize; ++i) { + STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); + + taosArrayPush(pTable, &pInfo->pTableObj); + } + + *psTable = pTable; + return pTableCheckInfo; } -static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey) { +static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey, SArray** psTable) { size_t si = taosArrayGetSize(pTableCheckInfo); SArray* pNew = taosArrayInit(si, sizeof(STableCheckInfo)); if (pNew == NULL) { return NULL; } + SArray* pTable = taosArrayInit(si, sizeof(STable*)); + for (int32_t j = 0; j < si; ++j) { STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, j); STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj}; info.tableId = pCheckInfo->tableId; taosArrayPush(pNew, &info); + taosArrayPush(pTable, &pCheckInfo->pTableObj); } + *psTable = pTable; + // it is ordered already, no need to sort again. taosArraySort(pNew, tsdbCheckInfoCompar); return pNew; } -static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) { +static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) { STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); if (pQueryHandle == NULL) { goto out_of_memory; @@ -282,7 +341,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; + pQueryHandle->cur.fid = INT32_MIN; pQueryHandle->cur.win = TSWINDOW_INITIALIZER; pQueryHandle->checkFiles = true; pQueryHandle->activeIndex = 0; // current active table index @@ -291,12 +350,13 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pQueryHandle->allocSize = 0; pQueryHandle->locateStart = false; pQueryHandle->pMemRef = pMemRef; + pQueryHandle->loadExternalRow = pCond->loadExternalRows; - if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) { + if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { goto out_of_memory; } - tsdbMayTakeMemSnapshot(pQueryHandle); + //tsdbMayTakeMemSnapshot(pQueryHandle); assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL); if (ASCENDING_TRAVERSE(pCond->order)) { @@ -351,26 +411,30 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* return NULL; } -TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) { +TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) { STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef); STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); + SArray* psTable = NULL; + // todo apply the lastkey of table check to avoid to load header file - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta); + pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); if (pQueryHandle->pTableCheckInfo == NULL) { tsdbCleanupQueryHandle(pQueryHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } + tsdbMayTakeMemSnapshot(pQueryHandle, psTable); + tsdbDebug("%p total numOfTable:%" PRIzu " in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); return (TsdbQueryHandleT) pQueryHandle; } -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) { - pCond->twindow = changeTableGroupByLastrow(groupList); +TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) { + pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table if (groupList->numOfTables == 0) { @@ -378,8 +442,14 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab } STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef); + int32_t code = checkForCachedLastRow(pQueryHandle, groupList); + if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 + terrno = code; + return NULL; + } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); + pQueryHandle->type = TSDB_QUERY_TYPE_LAST; return pQueryHandle; } @@ -399,12 +469,13 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { return res; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) { +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) { STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef); + pQueryHandle->loadExternalRow = true; if (pQueryHandle != NULL) { - pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; changeQueryHandleForInterpQuery(pQueryHandle); } + return pQueryHandle; } @@ -435,16 +506,18 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { pMem = pMemT->tData[pCheckInfo->tableId.tid]; if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid + TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); pCheckInfo->iter = - tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid + TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); pCheckInfo->iiter = - tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } @@ -609,7 +682,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); - pHandle->cur.fid = -1; + pHandle->cur.fid = INT32_MIN; STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); @@ -669,7 +742,7 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio return (int32_t)fid; } -static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { +static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { int32_t firstSlot = 0; int32_t lastSlot = numOfBlocks - 1; @@ -707,16 +780,16 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); pCheckInfo->numOfBlocks = 0; - if (tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb) != TSDB_CODE_SUCCESS) { + if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { code = terrno; break; } - SCompIdx* compIndex = &pQueryHandle->rhelper.curCompIdx; + SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; // no data block in this file, try next file - if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) { - continue; // no data blocks in the file belongs to pCheckInfo->pTable + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + continue; // no data blocks in the file belongs to pCheckInfo->pTable } if (pCheckInfo->compSize < (int32_t)compIndex->len) { @@ -729,12 +802,12 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo break; } - pCheckInfo->pCompInfo = (SCompInfo*) t; + pCheckInfo->pCompInfo = (SBlockInfo*) t; pCheckInfo->compSize = compIndex->len; } - tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); - SCompInfo* pCompInfo = pCheckInfo->pCompInfo; + tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); + SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; @@ -763,7 +836,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo pCheckInfo->numOfBlocks = (end - start); if (start > 0) { - memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SCompBlock)); + memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); } (*numOfBlocks) += pCheckInfo->numOfBlocks; @@ -772,7 +845,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return code; } -static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { +static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { int64_t st = taosGetTimestampUs(); STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); @@ -783,14 +856,14 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema); + code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %p", pQueryHandle, pQueryHandle->qinfo); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema); + code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %p", pQueryHandle, pQueryHandle->qinfo); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -812,7 +885,7 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p pBlockLoadInfo->slot = pQueryHandle->cur.slot; pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); pBlock->numOfRows = pCols->numOfRows; @@ -838,7 +911,7 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); -static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ +static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ SQueryFilePos* cur = &pQueryHandle->cur; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); @@ -921,7 +994,7 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc return code; } -static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { +static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { SQueryFilePos* cur = &pQueryHandle->cur; int32_t code = TSDB_CODE_SUCCESS; @@ -933,7 +1006,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); if (pCheckInfo->lastKey > pBlock->keyFirst) { @@ -956,9 +1029,9 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } @@ -1041,7 +1114,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity char* pData = NULL; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t num = end - start + 1; @@ -1074,7 +1147,12 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity if (pColInfo->info.colId == src->colId) { - if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { + if (pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + for (int32_t n = 0; n < num; n++) { + TKEY tkey = *(TKEY *)((char*)src->pData + bytes * start + n * sizeof(TKEY)); + *(TSKEY *)(pData + n * sizeof(TSKEY)) = tdGetKey(tkey); + } + } else if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { memmove(pData, (char*)src->pData + bytes * start, bytes * num); } else { // handle the var-string char* dst = pData; @@ -1133,13 +1211,18 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity } static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row, - int32_t numOfCols, STable* pTable) { + int32_t numOfCols, STable* pTable, STSchema* pSchema) { char* pData = NULL; // the schema version info is embeded in SDataRow - STSchema* pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - int32_t numOfRowCols = schemaNCols(pSchema); - + int32_t numOfRowCols = 0; + if (pSchema == NULL) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + numOfRowCols = schemaNCols(pSchema); + } else { + numOfRowCols = schemaNCols(pSchema); + } + int32_t i = 0, j = 0; while(i < numOfCols && j < numOfRowCols) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); @@ -1156,10 +1239,40 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if (pSchema->columns[j].colId == pColInfo->info.colId) { void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - memcpy(pData, value, varDataTLen(value)); - } else { - memcpy(pData, value, pColInfo->info.bytes); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *)pData = *(uint8_t *)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t *)pData = *(uint16_t *)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t *)pData = *(uint32_t *)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t *)pData = *(uint64_t *)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + *(TSKEY *)pData = tdGetKey(*(TKEY *)value); + break; + default: + memcpy(pData, value, pColInfo->info.bytes); } j++; @@ -1265,7 +1378,7 @@ static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; @@ -1308,7 +1421,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { endPos = pBlockInfo->rows - 1; @@ -1327,14 +1440,14 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; initTableMemIterator(pQueryHandle, pCheckInfo); - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && cur->pos >= 0 && cur->pos < pBlock->numOfRows); @@ -1358,6 +1471,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; + int16_t rv = -1; + STSchema* pSchema = NULL; + int32_t pos = cur->pos; cur->win = TSWINDOW_INITIALIZER; @@ -1379,14 +1495,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* break; } - if (((tsArray[pos] > pQueryHandle->window.ekey || pos > endPos) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((tsArray[pos] < pQueryHandle->window.ekey || pos < endPos) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { break; } if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1399,7 +1520,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1626,7 +1752,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO continue; } - SCompBlock* pBlock = pTableCheck->pCompInfo->blocks; + SBlock* pBlock = pTableCheck->pCompInfo->blocks; sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; char* buf = calloc(1, sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); @@ -1704,7 +1830,32 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO return TSDB_CODE_SUCCESS; } -static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) { +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); + +static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + SQueryFilePos* cur = &pQueryHandle->cur; + + while(1) { + int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + + if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + // all data blocks in current file has been checked already, try next file if exists + return getFirstFileDataBlock(pQueryHandle, exists); + } else { // next block of the same file + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; + pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + } + } +} + +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { pQueryHandle->numOfBlocks = 0; SQueryFilePos* cur = &pQueryHandle->cur; @@ -1717,19 +1868,19 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex STimeWindow win = TSWINDOW_INITIALIZER; while (true) { - pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) == NULL) { - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo); pQueryHandle->pFileGroup = NULL; @@ -1737,15 +1888,15 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex break; } - if (tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); code = terrno; break; } - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - if (tsdbLoadCompIdx(&pQueryHandle->rhelper, NULL) < 0) { + if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { code = terrno; break; } @@ -1755,7 +1906,7 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex } tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo); + pQueryHandle->pFileGroup->fid, pQueryHandle->qinfo); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -1779,35 +1930,51 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex assert(pQueryHandle->pFileGroup == NULL); } - cur->fid = -1; // denote that there are no data in file anymore + cur->fid = INT32_MIN; // denote that there are no data in file anymore *exists = false; return code; } assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fileId; + cur->fid = pQueryHandle->pFileGroup->fid; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo, exists); + return getDataBlockRv(pQueryHandle, pBlockInfo, exists); +} + +static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { + assert(cur != NULL && numOfBlocks > 0); + return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); +} + +static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + + SQueryFilePos* cur = &pQueryHandle->cur; + assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; } static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { - STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); + STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); SQueryFilePos* cur = &pQueryHandle->cur; // find the start data block in file if (!pQueryHandle->locateStart) { pQueryHandle->locateStart = true; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); - pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); - tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); - tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + tsdbRLockFS(pFileHandle); + tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); + tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbUnLockFS(pFileHandle); - return getDataBlocksInFilesImpl(pQueryHandle, exists); + return getFirstFileDataBlock(pQueryHandle, exists); } else { // check if current file block is all consumed STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -1815,27 +1982,26 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - // all data blocks in current file has been checked already, try next file if exists - return getDataBlocksInFilesImpl(pQueryHandle, exists); - } else { - // next block of the same file - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->slot += step; - - cur->mixBlock = false; - cur->blockCompleted = false; - - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); - } + // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, pQueryHandle->qinfo); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, + pQueryHandle->qinfo); int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = pQueryHandle->realNumOfRows > 0; + *exists = (pQueryHandle->realNumOfRows > 0); - return code; + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + } + + // current block is empty, try next block in file + // all data blocks in current file has been checked already, try next file if exists + if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { + return getFirstFileDataBlock(pQueryHandle, exists); + } else { + moveToNextDataBlockInCurrentFile(pQueryHandle); + STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pQueryHandle, pNext, exists); } } } @@ -1851,17 +2017,21 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { pQueryHandle->activeIndex += 1; } + if (pQueryHandle->loadExternalRow && pQueryHandle->window.skey == pQueryHandle->window.ekey) { + SMemRef* pMemRef = pQueryHandle->pMemRef; + doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); + doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); + } + // no data in memtable or imemtable, decrease the memory reference. tsdbMayUnTakeMemSnapshot(pQueryHandle); return false; } +//todo not unref yet, since it is not support multi-group interpolation query static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { // filter the queried time stamp in the first place STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - pQueryHandle->order = TSDB_ORDER_DESC; - - assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); // starts from the buffer in case of descending timestamp order check data blocks size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); @@ -1871,8 +2041,8 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); // the first qualified table for interpolation query - if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && - pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { + if ((pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && + (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { break; } @@ -1889,9 +2059,6 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { info.lastKey = pQueryHandle->window.skey; taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; } static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, @@ -1903,6 +2070,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int int64_t st = taosGetTimestampUs(); STable* pTable = pCheckInfo->pTableObj; + int16_t rv = -1; + STSchema* pSchema = NULL; do { SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); @@ -1923,7 +2092,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -1971,7 +2144,6 @@ static void destroyHelper(void* param) { return; } - tQueryInfo* pInfo = (tQueryInfo*)param; if (pInfo->optr != TSDB_RELATION_IN) { tfree(pInfo->q); @@ -1980,113 +2152,82 @@ static void destroyHelper(void* param) { free(param); } -static bool getNeighborRows(STsdbQueryHandle* pQueryHandle) { - assert (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL); +// handle data in cache situation +bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - SDataBlockInfo blockInfo = {{0}, 0}; + int64_t stime = taosGetTimestampUs(); + int64_t elapsedTime = stime; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->order = TSDB_ORDER_DESC; + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables > 0); - if (!tsdbNextDataBlock((void*) pQueryHandle)) { - return false; - } + if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { + // the last row is cached in buffer, return it directly. + // here note that the pQueryHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + SQueryFilePos* cur = &pQueryHandle->cur; + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + + if (++pQueryHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); + if (ret != TSDB_CODE_SUCCESS) { + return false; + } - tsdbRetrieveDataBlockInfo((void*) pQueryHandle, &blockInfo); - /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pQueryHandle, pQueryHandle->defaultLoadColumn); - if (terrno != TSDB_CODE_SUCCESS) { - return false; - } + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); + tfree(pRow); - if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) { - // data already retrieve, discard other data rows and return - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes); - } + // update the last key value + pCheckInfo->lastKey = key + step; - pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey}; - pQueryHandle->window = pQueryHandle->cur.win; - pQueryHandle->cur.rows = 1; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - return true; - } else { - STimeWindow win = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX}; - STsdbQueryCond cond = { - .order = TSDB_ORDER_ASC, - .numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)) - }; - cond.twindow = win; - - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - if (cond.colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return false; - } + cur->rows = 1; // only one row + cur->lastKey = key + step; + cur->mixBlock = true; + cur->win.skey = key; + cur->win.ekey = key; - for(int32_t i = 0; i < cond.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); + return true; } - STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef); + return false; + } - tfree(cond.colList); + if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block + bool exists = true; - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey); - if (pSecQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pSecQueryHandle); - return false; - } + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; - if (!tsdbNextDataBlock((void*) pSecQueryHandle)) { - tsdbCleanupQueryHandle(pSecQueryHandle); return false; } - tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo); - tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); - - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pSecQueryHandle)); - size_t si = taosArrayGetSize(pSecQueryHandle->pTableCheckInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes); - - SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i); - assert(pCol->info.colId == pCol1->info.colId); - - memcpy((char*)pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes); - } - - SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0); - - // it is ascending order - pQueryHandle->order = TSDB_ORDER_DESC; - pQueryHandle->window = pQueryHandle->cur.win; - pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]}; - pQueryHandle->cur.rows = 2; - pQueryHandle->cur.mixBlock = true; - - int32_t step = -1;// one step for ascending order traverse - for (int32_t j = 0; j < si; ++j) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j); - pCheckInfo->lastKey = pQueryHandle->cur.win.ekey + step; + if (exists) { + pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + return exists; } - tsdbCleanupQueryHandle(pSecQueryHandle); + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; } - //disable it after retrieve data - pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; - pQueryHandle->checkFiles = false; - return true; + // TODO: opt by consider the scan order + bool ret = doHasDataInBuffer(pQueryHandle); + terrno = TSDB_CODE_SUCCESS; + + elapsedTime = taosGetTimestampUs() - stime; + pQueryHandle->cost.checkForNextTime += elapsedTime; + return ret; } -// handle data in cache situation -bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { +bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT* pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; int64_t stime = taosGetTimestampUs(); @@ -2095,15 +2236,39 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); assert(numOfTables > 0); - if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) { - SMemRef* pMemRef = pQueryHandle->pMemRef; - tsdbMayTakeMemSnapshot(pQueryHandle); - bool ret = getNeighborRows(pQueryHandle); - tsdbMayUnTakeMemSnapshot(pQueryHandle); + if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { + // the last row is cached in buffer, return it directly. + // here note that the pQueryHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + SQueryFilePos* cur = &pQueryHandle->cur; + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + + if (++pQueryHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); + if (ret != TSDB_CODE_SUCCESS) { + return false; + } - // restore the pMemRef - pQueryHandle->pMemRef = pMemRef; - return ret; + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); + tfree(pRow); + + // update the last key value + pCheckInfo->lastKey = key + step; + + cur->rows = 1; // only one row + cur->lastKey = key + step; + cur->mixBlock = true; + cur->win.skey = key; + cur->win.ekey = key; + + return true; + } + + return false; } if (pQueryHandle->checkFiles) { @@ -2127,16 +2292,177 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pQueryHandle->checkFiles = false; } - // TODO: opt by consider the scan order - bool ret = doHasDataInBuffer(pQueryHandle); - terrno = TSDB_CODE_SUCCESS; - elapsedTime = taosGetTimestampUs() - stime; pQueryHandle->cost.checkForNextTime += elapsedTime; - return ret; + return false; } -STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { +static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { + STsdbQueryHandle* pSecQueryHandle = NULL; + + if (type == TSDB_PREV_ROW && pQueryHandle->prev) { + return TSDB_CODE_SUCCESS; + } + + if (type == TSDB_NEXT_ROW && pQueryHandle->next) { + return TSDB_CODE_SUCCESS; + } + + // prepare the structure + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); + + if (type == TSDB_PREV_ROW) { + pQueryHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pQueryHandle->prev == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + } else { + pQueryHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pQueryHandle->next == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + } + + SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + + SColumnInfoData colInfo = {{0}, 0}; + colInfo.info = pCol->info; + colInfo.pData = calloc(1, pCol->info.bytes); + if (colInfo.pData == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + + taosArrayPush(row, &colInfo); + } + + // load the previous row + STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false,}; + if (type == TSDB_PREV_ROW) { + cond.order = TSDB_ORDER_DESC; + cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; + } else { + cond.order = TSDB_ORDER_ASC; + cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MAX}; + } + + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + if (cond.colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + + for (int32_t i = 0; i < cond.numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); + memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); + } + + pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pMemRef); + + tfree(cond.colList); + + SArray* psTable = NULL; + + pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey, &psTable); + if (pSecQueryHandle->pTableCheckInfo == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + + + tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); + + if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { + // no result in current query, free the corresponding result rows structure + if (type == TSDB_PREV_ROW) { + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + } else { + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + } + + goto out_of_memory; + } + + SDataBlockInfo blockInfo = {{0}, 0}; + tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); + tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); + + row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; + int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(row, i); + SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); + memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); + } + +out_of_memory: + tsdbCleanupQueryHandle(pSecQueryHandle); + return terrno; +} + +SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t type) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + assert(type == TSDB_PREV_ROW || type == TSDB_NEXT_ROW); + return (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; +} + +/* + * 1. no data at all (pTable->lastKey = TSKEY_INITIAL_VAL), just return TSKEY_INITIAL_VAL + * 2. has data but not loaded, just return lastKey but not set pRes + * 3. has data and loaded, return lastKey and set pRes + */ +int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { + TSDB_RLOCK_TABLE(pTable); + *lastKey = pTable->lastKey; + + if ((*lastKey) != TSKEY_INITIAL_VAL && pTable->lastRow) { + *pRes = tdDataRowDup(pTable->lastRow); + if (*pRes == NULL) { + TSDB_RUNLOCK_TABLE(pTable); + return TSDB_CODE_TDB_OUT_OF_MEMORY; + } + } + + TSDB_RUNLOCK_TABLE(pTable); + return TSDB_CODE_SUCCESS; +} + +int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { + assert(pQueryHandle != NULL && groupList != NULL); + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + + SArray* group = taosArrayGetP(groupList->pGroupList, 0); + assert(group != NULL); + + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); + + int32_t code = tsdbGetCachedLastRow(pInfo->pTable, &pRow, &key); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->cachelastrow = false; + } else { + pQueryHandle->cachelastrow = (pRow != NULL); + } + + // update the tsdb query time range + if (pQueryHandle->cachelastrow) { + pQueryHandle->window = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = false; + pQueryHandle->activeIndex = -1; // start from -1 + } + + tfree(pRow); + return code; +} + +STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; int32_t totalNumOfTable = 0; @@ -2151,16 +2477,16 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { size_t numOfTables = taosArrayGetSize(pGroup); for(int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); // if the lastKey equals to INT64_MIN, there is no data in this table - TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; - keyInfo.pTable = pKeyInfo->pTable; + keyInfo.pTable = pInfo->pTable; keyInfo.lastKey = key; - pKeyInfo->lastKey = key; + pInfo->lastKey = key; if (key < window.skey) { window.skey = key; @@ -2174,11 +2500,11 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { // clear current group, unref unused table for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); // keyInfo.pTable may be NULL here. - if (pKeyInfo->pTable != keyInfo.pTable) { - tsdbUnRefTable(pKeyInfo->pTable); + if (pInfo->pTable != keyInfo.pTable) { + tsdbUnRefTable(pInfo->pTable); } } @@ -2213,7 +2539,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p STable* pTable = NULL; // there are data in file - if (pHandle->cur.fid >= 0) { + if (pHandle->cur.fid != INT32_MIN) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; pTable = pBlockInfo->pTableCheckInfo->pTableObj; } else { @@ -2250,7 +2576,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta } int64_t stime = taosGetTimestampUs(); - tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL); + tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock); int16_t* colIds = pHandle->defaultLoadColumn->pData; @@ -2260,7 +2586,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta pHandle->statis[i].colId = colIds[i]; } - tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols); + tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols); // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; @@ -2297,7 +2623,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { */ STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; - if (pHandle->cur.fid < 0) { + if (pHandle->cur.fid == INT32_MIN) { return pHandle->pColumns; } else { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; @@ -2312,11 +2638,11 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; - if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid && + if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { return pHandle->pColumns; } else { // only load the file block - SCompBlock* pBlock = pBlockInfo->compBlock; + SBlock* pBlock = pBlockInfo->compBlock; if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) { return NULL; } @@ -2392,7 +2718,7 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa f1 = (char*) TABLE_NAME(pTable1); f2 = (char*) TABLE_NAME(pTable2); type = TSDB_DATA_TYPE_BINARY; - bytes = tGetTableNameColumnSchema().bytes; + bytes = tGetTbnameColumnSchema()->bytes; } else { STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); bytes = pCol->bytes; @@ -2513,13 +2839,12 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -static bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool tableFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = NULL; - + char* val = NULL; if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { val = (char*) TABLE_NAME(pTable); } else { @@ -2574,20 +2899,22 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) { return true; } +static void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the expression tree SExprTraverseSupp supp = { - .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp, + .nodeFilterFn = (__result_filter_fn_t) tableFilterFp, .setupInfoFn = filterPrepare, .pExtInfo = pSTable->tagSchema, }; - tExprTreeTraverse(pExpr, pSTable->pIndex, pRes, &supp); - tExprTreeDestroy(&pExpr, destroyHelper); + getTableListfromSkipList(pExpr, pSTable->pIndex, pRes, &supp); + tExprTreeDestroy(pExpr, destroyHelper); return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -2640,10 +2967,10 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co if (expr == NULL) { expr = exprTreeFromBinary(pTagCond, len); } else { - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL); tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); if (tagExpr != NULL) { - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, tagExpr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, tagExpr, NULL); tExprNode* tbnameExpr = expr; expr = calloc(1, sizeof(tExprNode)); if (expr == NULL) { @@ -2682,7 +3009,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co return terrno; } -int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -2712,7 +3039,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, ST return terrno; } -int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; } @@ -2757,6 +3084,21 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa return TSDB_CODE_SUCCESS; } +static void* doFreeColumnInfoData(SArray* pColumnInfoData) { + if (pColumnInfoData == NULL) { + return NULL; + } + + size_t cols = taosArrayGetSize(pColumnInfoData); + for (int32_t i = 0; i < cols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pColumnInfoData, i); + tfree(pColInfo->pData); + } + + taosArrayDestroy(pColumnInfoData); + return NULL; +} + void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; if (pQueryHandle == NULL) { @@ -2774,14 +3116,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { taosArrayDestroy(pQueryHandle->pTableCheckInfo); } - if (pQueryHandle->pColumns != NULL) { - size_t cols = taosArrayGetSize(pQueryHandle->pColumns); - for (int32_t i = 0; i < cols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - tfree(pColInfo->pData); - } - taosArrayDestroy(pQueryHandle->pColumns); - } + pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); taosArrayDestroy(pQueryHandle->defaultLoadColumn); tfree(pQueryHandle->pDataBlockInfo); @@ -2790,11 +3125,14 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { // todo check error tsdbMayUnTakeMemSnapshot(pQueryHandle); - tsdbDestroyHelper(&pQueryHandle->rhelper); + tsdbDestroyReadH(&pQueryHandle->rhelper); tdFreeDataCols(pQueryHandle->pDataCols); pQueryHandle->pDataCols = NULL; + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + SIOCostSummary* pCost = &pQueryHandle->cost; tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); @@ -2824,3 +3162,235 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); pGroupList->numOfTables = 0; } + +static void applyFilterToSkipListNode(SSkipList *pSkipList, tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + // Scan each node in the skiplist by using iterator + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + if (exprTreeApplayFilter(pExpr, pNode, param)) { + taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); + } + } + + tSkipListDestroyIter(iter); +} + +typedef struct { + char* v; + int32_t optr; +} SEndPoint; + +typedef struct { + SEndPoint* start; + SEndPoint* end; +} SQueryCond; + +// todo check for malloc failure +static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { + int32_t optr = queryColInfo->optr; + + if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || + optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { + pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start->optr = queryColInfo->optr; + pCond->start->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + pCond->end = calloc(1, sizeof(SEndPoint)); + pCond->end->optr = queryColInfo->optr; + pCond->end->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { + assert(0); + } + + return TSDB_CODE_SUCCESS; +} + +static void queryIndexedColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = NULL; + + SQueryCond cond = {0}; + if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { + //todo handle error + } + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); + } else { + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); + } + + if (cond.start != NULL) { + int32_t optr = cond.start->optr; + + if (optr == TSDB_RELATION_EQUAL) { // equals + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + if (ret != 0) { + break; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + assert(ret >= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_GREATER) { + continue; + } else { + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; + } + } + } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal + bool comp = true; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + tSkipListDestroyIter(iter); + + comp = true; + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + } else { + assert(0); + } + } else { + int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; + if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); + assert(ret <= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_LESS) { + continue; + } else { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; // no need to compare anymore + } + } + } else { + assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); + if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || + (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } + } + } + + free(cond.start); + free(cond.end); + tSkipListDestroyIter(iter); +} + +static void queryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + + char *pData = SL_GET_NODE_DATA(pNode); + tstr *name = (tstr*) tsdbGetTableName((void*) pData); + + // todo speed up by using hash + if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(name, pQueryInfo->q); + } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = !pQueryInfo->compare(name, pQueryInfo->q); + } + } else { + addToResult = filterFp(pNode, pQueryInfo); + } + + if (addToResult) { + STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(res, &info); + } + } + + tSkipListDestroyIter(iter); +} + +// Apply the filter expression to each node in the skiplist to acquire the qualified nodes in skip list +void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { + if (pExpr == NULL) { + return; + } + + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { + assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); + + param->setupInfoFn(pExpr, param->pExtInfo); + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { + queryIndexedColumn(pSkipList, pQueryInfo, result); + } else { + queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); + } + + return; + } + + // The value of hasPK is always 0. + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); + + //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes + applyFilterToSkipListNode(pSkipList, pExpr, result, param); +} diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c new file mode 100644 index 0000000000000000000000000000000000000000..572706d45e6a085f44ffd0a089dc28256255e33e --- /dev/null +++ b/src/tsdb/src/tsdbReadImpl.c @@ -0,0 +1,660 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbint.h" + +#define TSDB_KEY_COL_OFFSET 0 + +static void tsdbResetReadTable(SReadH *pReadh); +static void tsdbResetReadFile(SReadH *pReadh); +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, + int maxPoints, char *buffer, int bufferSize); +static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds, + int numOfColIds); +static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); + +int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) { + ASSERT(pReadh != NULL && pRepo != NULL); + + STsdbCfg *pCfg = REPO_CFG(pRepo); + + memset((void *)pReadh, 0, sizeof(*pReadh)); + pReadh->pRepo = pRepo; + + TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); + + pReadh->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pReadh->aBlkIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + pReadh->pDCols[0] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + if (pReadh->pDCols[0] == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyReadH(pReadh); + return -1; + } + + pReadh->pDCols[1] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + if (pReadh->pDCols[1] == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyReadH(pReadh); + return -1; + } + + return 0; +} + +void tsdbDestroyReadH(SReadH *pReadh) { + if (pReadh == NULL) return; + + pReadh->pCBuf = taosTZfree(pReadh->pCBuf); + pReadh->pBuf = taosTZfree(pReadh->pBuf); + pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]); + pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]); + pReadh->pBlkData = taosTZfree(pReadh->pBlkData); + pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo); + pReadh->cidx = 0; + pReadh->pBlkIdx = NULL; + pReadh->pTable = NULL; + pReadh->aBlkIdx = taosArrayDestroy(pReadh->aBlkIdx); + tsdbCloseDFileSet(TSDB_READ_FSET(pReadh)); + pReadh->pRepo = NULL; +} + +int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { + ASSERT(pSet != NULL); + tsdbResetReadFile(pReadh); + + pReadh->rSet = *pSet; + TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); + if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), + tstrerror(terrno)); + return -1; + } + + return 0; +} + +void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); } + +int tsdbLoadBlockIdx(SReadH *pReadh) { + SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SBlockIdx blkIdx; + + ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0); + + // No data at all, just return + if (pHeadf->info.offset <= 0) return 0; + + if (tsdbSeekDFile(pHeadf, pHeadf->info.offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset, + pHeadf->info.len); + return -1; + } + + if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pHeadf->info.len) < 0) return -1; + + int64_t nread = tsdbReadDFile(pHeadf, TSDB_READ_BUF(pReadh), pHeadf->info.len); + if (nread < 0) { + tsdbError("vgId:%d failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset, + pHeadf->info.len); + return -1; + } + + if (nread < pHeadf->info.len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len, nread); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), pHeadf->info.len)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len); + return -1; + } + + void *ptr = TSDB_READ_BUF(pReadh); + int tsize = 0; + while (POINTER_DISTANCE(ptr, TSDB_READ_BUF(pReadh)) < (pHeadf->info.len - sizeof(TSCKSUM))) { + ptr = tsdbDecodeSBlockIdx(ptr, &blkIdx); + ASSERT(ptr != NULL); + + if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + tsize++; + ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid < + ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid); + } + + return 0; +} + +int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { + STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + + pReadh->pTable = pTable; + + if (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + size_t size = taosArrayGetSize(pReadh->aBlkIdx); + if (size > 0) { + while (true) { + if (pReadh->cidx >= size) { + pReadh->pBlkIdx = NULL; + break; + } + + SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx); + if (pBlkIdx->tid == TABLE_TID(pTable)) { + if (pBlkIdx->uid == TABLE_UID(pTable)) { + pReadh->pBlkIdx = pBlkIdx; + } else { + pReadh->pBlkIdx = NULL; + } + pReadh->cidx++; + break; + } else if (pBlkIdx->tid > TABLE_TID(pTable)) { + pReadh->pBlkIdx = NULL; + break; + } else { + pReadh->cidx++; + } + } + } else { + pReadh->pBlkIdx = NULL; + } + + return 0; +} + +int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { + ASSERT(pReadh->pBlkIdx != NULL); + + SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SBlockIdx *pBlkIdx = pReadh->pBlkIdx; + + if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len); + return -1; + } + + if (tsdbMakeRoom((void **)(&(pReadh->pBlkInfo)), pBlkIdx->len) < 0) return -1; + + int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len); + if (nread < 0) { + tsdbError("vgId:%d failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len); + return -1; + } + + if (nread < pBlkIdx->len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len); + return -1; + } + + ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid); + + if (pTarget) { + memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len); + } + + return 0; +} + +int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { + ASSERT(pBlock->numOfSubBlocks > 0); + + SBlock *iBlock = pBlock; + if (pBlock->numOfSubBlocks > 1) { + if (pBlkInfo) { + iBlock = (SBlock *)POINTER_SHIFT(pBlkInfo, pBlock->offset); + } else { + iBlock = (SBlock *)POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset); + } + } + + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1; + for (int i = 1; i < pBlock->numOfSubBlocks; i++) { + iBlock++; + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; + if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1; + } + + ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + + return 0; +} + +int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds) { + ASSERT(pBlock->numOfSubBlocks > 0); + + SBlock *iBlock = pBlock; + if (pBlock->numOfSubBlocks > 1) { + if (pBlkInfo) { + iBlock = POINTER_SHIFT(pBlkInfo, pBlock->offset); + } else { + iBlock = POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset); + } + } + + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1; + for (int i = 1; i < pBlock->numOfSubBlocks; i++) { + iBlock++; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; + if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1; + } + + ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + + return 0; +} + +int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { + ASSERT(pBlock->numOfSubBlocks <= 1); + + SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + + if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); + return -1; + } + + size_t size = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); + if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1; + + int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); + if (nread < 0) { + tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size); + return -1; + } + + if (nread < size) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu + " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); + return -1; + } + + return 0; +} + +int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) { + int tlen = 0; + + tlen += taosEncodeVariantI32(buf, pIdx->tid); + tlen += taosEncodeVariantU32(buf, pIdx->len); + tlen += taosEncodeVariantU32(buf, pIdx->offset); + tlen += taosEncodeFixedU8(buf, pIdx->hasLast); + tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); + tlen += taosEncodeFixedU64(buf, pIdx->uid); + tlen += taosEncodeFixedU64(buf, pIdx->maxKey); + + return tlen; +} + +void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) { + uint8_t hasLast = 0; + uint32_t numOfBlocks = 0; + uint64_t value = 0; + + if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; + if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; + pIdx->hasLast = hasLast; + if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; + pIdx->numOfBlocks = numOfBlocks; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->uid = (int64_t)value; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->maxKey = (TSKEY)value; + + return buf; +} + +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) { + SBlockData *pBlockData = pReadh->pBlkData; + + for (int i = 0, j = 0; i < numOfCols;) { + if (j >= pBlockData->numOfCols) { + pStatis[i].numOfNull = -1; + i++; + continue; + } + + if (pStatis[i].colId == pBlockData->cols[j].colId) { + pStatis[i].sum = pBlockData->cols[j].sum; + pStatis[i].max = pBlockData->cols[j].max; + pStatis[i].min = pBlockData->cols[j].min; + pStatis[i].maxIndex = pBlockData->cols[j].maxIndex; + pStatis[i].minIndex = pBlockData->cols[j].minIndex; + pStatis[i].numOfNull = pBlockData->cols[j].numOfNull; + i++; + j++; + } else if (pStatis[i].colId < pBlockData->cols[j].colId) { + pStatis[i].numOfNull = -1; + i++; + } else { + j++; + } + } +} + +static void tsdbResetReadTable(SReadH *pReadh) { + tdResetDataCols(pReadh->pDCols[0]); + tdResetDataCols(pReadh->pDCols[1]); + pReadh->cidx = 0; + pReadh->pBlkIdx = NULL; + pReadh->pTable = NULL; +} + +static void tsdbResetReadFile(SReadH *pReadh) { + tsdbResetReadTable(pReadh); + taosArrayClear(pReadh->aBlkIdx); + tsdbCloseDFileSet(TSDB_READ_FSET(pReadh)); +} + +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) { + ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); + + SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + + tdResetDataCols(pDataCols); + if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1; + + SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh); + + if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block data part while seek file %s to offset %" PRId64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); + return -1; + } + + int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlock->len); + if (nread < 0) { + tsdbError("vgId:%d failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, + pBlock->len); + return -1; + } + + if (nread < pBlock->len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block data part in file %s is corrupted, offset:%" PRId64 + " expected bytes:%d read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, pBlock->len, nread); + return -1; + } + + int32_t tsize = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); + if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize); + return -1; + } + + ASSERT(tsize < pBlock->len); + ASSERT(pBlockData->numOfCols == pBlock->numOfCols); + + pDataCols->numOfRows = pBlock->numOfRows; + + // Recover the data + int ccol = 0; // loop iter for SBlockCol object + int dcol = 0; // loop iter for SDataCols object + while (dcol < pDataCols->numOfCols) { + SDataCol *pDataCol = &(pDataCols->cols[dcol]); + if (dcol != 0 && ccol >= pBlockData->numOfCols) { + // Set current column as NULL and forward + dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + dcol++; + continue; + } + + int16_t tcolId = 0; + uint32_t toffset = TSDB_KEY_COL_OFFSET; + int32_t tlen = pBlock->keyLen; + + if (dcol != 0) { + SBlockCol *pBlockCol = &(pBlockData->cols[ccol]); + tcolId = pBlockCol->colId; + toffset = tsdbGetBlockColOffset(pBlockCol); + tlen = pBlockCol->len; + } else { + ASSERT(pDataCol->colId == tcolId); + } + + if (tcolId == pDataCol->colId) { + if (pBlock->algorithm == TWO_STAGE_COMP) { + int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1; + } + + if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, + pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), + (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); + return -1; + } + + if (dcol != 0) { + ccol++; + } + dcol++; + } else if (tcolId < pDataCol->colId) { + ccol++; + } else { + // Set current column as NULL and forward + dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + dcol++; + } + } + + return 0; +} + +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, + int maxPoints, char *buffer, int bufferSize) { + if (!taosCheckChecksumWhole((uint8_t *)content, len)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + // Decode the data + if (comp) { + // Need to decompress + int tlen = (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, + pDataCol->spaceSize, comp, buffer, bufferSize); + if (tlen <= 0) { + tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", + len, comp, numOfRows, maxPoints, bufferSize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + pDataCol->len = tlen; + } else { + // No need to decompress, just memcpy it + pDataCol->len = len - sizeof(TSCKSUM); + memcpy(pDataCol->pData, content, pDataCol->len); + } + + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + dataColSetOffset(pDataCol, numOfRows); + } + return 0; +} + +static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds, + int numOfColIds) { + ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); + ASSERT(colIds[0] == 0); + + SDFile * pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + SBlockCol blockCol = {0}; + + tdResetDataCols(pDataCols); + + // If only load timestamp column, no need to load SBlockData part + if (numOfColIds > 1 && tsdbLoadBlockStatis(pReadh, pBlock) < 0) return -1; + + pDataCols->numOfRows = pBlock->numOfRows; + + int dcol = 0; + int ccol = 0; + for (int i = 0; i < numOfColIds; i++) { + int16_t colId = colIds[i]; + SDataCol * pDataCol = NULL; + SBlockCol *pBlockCol = NULL; + + while (true) { + if (dcol >= pDataCols->numOfCols) { + pDataCol = NULL; + break; + } + pDataCol = &pDataCols->cols[dcol]; + if (pDataCol->colId > colId) { + pDataCol = NULL; + break; + } else { + dcol++; + if (pDataCol->colId == colId) break; + } + } + + if (pDataCol == NULL) continue; + ASSERT(pDataCol->colId == colId); + + if (colId == 0) { // load the key row + blockCol.colId = colId; + blockCol.len = pBlock->keyLen; + blockCol.type = pDataCol->type; + blockCol.offset = TSDB_KEY_COL_OFFSET; + pBlockCol = &blockCol; + } else { // load non-key rows + while (true) { + if (ccol >= pBlock->numOfCols) { + pBlockCol = NULL; + break; + } + + pBlockCol = &(pReadh->pBlkData->cols[ccol]); + if (pBlockCol->colId > colId) { + pBlockCol = NULL; + break; + } else { + ccol++; + if (pBlockCol->colId == colId) break; + } + } + + if (pBlockCol == NULL) { + dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + continue; + } + + ASSERT(pBlockCol->colId == pDataCol->colId); + } + + if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1; + } + + return 0; +} + +static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { + ASSERT(pDataCol->colId == pBlockCol->colId); + + STsdbRepo *pRepo = TSDB_READ_REPO(pReadh); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int tsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + + if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; + if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; + + int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + tsdbGetBlockColOffset(pBlockCol); + if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno)); + return -1; + } + + int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlockCol->len); + if (nread < 0) { + tsdbError("vgId:%d failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), offset, pBlockCol->len); + return -1; + } + + if (nread < pBlockCol->len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu + " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, pBlockCol->len, nread); + return -1; + } + + if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows, + pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) { + tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + pBlockCol->colId, offset); + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbScan.c b/src/tsdb/src/tsdbScan.c index 91f67878740c90d56a8c5c124be23c1f53e7da40..382f7b11ae021152c9c8d314b24428b51c2e107b 100644 --- a/src/tsdb/src/tsdbScan.c +++ b/src/tsdb/src/tsdbScan.c @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#include "tsdbMain.h" +#include "tsdbint.h" +#if 0 #ifndef _TSDB_PLUGINS int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } @@ -25,12 +26,13 @@ void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {} int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } -int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle) { return 0; } +int tsdbScanSBlockIdx(STsdbScanHandle* pScanHandle) { return 0; } -int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; } +int tsdbScanSBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; } int tsdbCloseScanFile(STsdbScanHandle* pScanHandle) { return 0; } void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle) {} +#endif #endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c new file mode 100644 index 0000000000000000000000000000000000000000..6b8483e4a66df75db529b05db9a29e800e01427a --- /dev/null +++ b/src/tsdb/src/tsdbSync.c @@ -0,0 +1,699 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tsdbint.h" + +// Sync handle +typedef struct { + STsdbRepo *pRepo; + SRtn rtn; + SOCKET socketFd; + void * pBuf; + bool mfChanged; + SMFile * pmf; + SMFile mf; + SDFileSet df; + SDFileSet *pdf; +} SSyncH; + +#define SYNC_BUFFER(sh) ((sh)->pBuf) + +static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd); +static void tsdbDestroySyncH(SSyncH *pSyncH); +static int32_t tsdbSyncSendMeta(SSyncH *pSynch); +static int32_t tsdbSyncRecvMeta(SSyncH *pSynch); +static int32_t tsdbSendMetaInfo(SSyncH *pSynch); +static int32_t tsdbRecvMetaInfo(SSyncH *pSynch); +static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend); +static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend); +static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch); +static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch); +static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2); +static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet); +static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet); +static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch); +static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged); + +int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) { + STsdbRepo *pRepo = (STsdbRepo *)tsdb; + SSyncH synch = {0}; + + tsdbInitSyncH(&synch, pRepo, socketFd); + // Disable TSDB commit + tsem_wait(&(pRepo->readyToCommit)); + + if (tsdbSyncSendMeta(&synch) < 0) { + tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + if (tsdbSyncSendDFileSetArray(&synch) < 0) { + tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + // Enable TSDB commit + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + return 0; + +_err: + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + return -1; +} + +int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { + STsdbRepo *pRepo = (STsdbRepo *)tsdb; + SSyncH synch = {0}; + + pRepo->state = TSDB_STATE_OK; + + tsdbInitSyncH(&synch, pRepo, socketFd); + tsem_wait(&(pRepo->readyToCommit)); + tsdbStartFSTxn(pRepo, 0, 0); + + if (tsdbSyncRecvMeta(&synch) < 0) { + tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + if (tsdbSyncRecvDFileSetArray(&synch) < 0) { + tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + tsdbEndFSTxn(pRepo); + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + + // Reload file change + tsdbReload(pRepo, synch.mfChanged); + + return 0; + +_err: + tsdbEndFSTxnWithError(REPO_FS(pRepo)); + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + return -1; +} + +static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) { + pSyncH->pRepo = pRepo; + pSyncH->socketFd = socketFd; + tsdbGetRtnSnap(pRepo, &(pSyncH->rtn)); +} + +static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); } + +static int32_t tsdbSyncSendMeta(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + bool toSendMeta = false; + SMFile mf; + + // Send meta info to remote + tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo)); + if (tsdbSendMetaInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (pRepo->fs->cstatus->pmf == NULL) { + // No meta file, not need to wait to retrieve meta file + tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo)); + return 0; + } + + if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) { + tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (toSendMeta) { + tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf); + if (tsdbOpenMFile(&mf, O_RDONLY) < 0) { + tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + int32_t writeLen = (int32_t)mf.info.size; + tsdbInfo("vgId:%d, metafile:%s will be sent, size:%d", REPO_ID(pRepo), mf.f.aname, writeLen); + + int32_t ret = (int32_t)taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send metafile since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret, + writeLen); + tsdbCloseMFile(&mf); + return -1; + } + + tsdbCloseMFile(&mf); + tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo)); + } else { + tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo)); + } + + return 0; +} + +static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + SMFile * pLMFile = pRepo->fs->cstatus->pmf; + + // Recv meta info from remote + if (tsdbRecvMetaInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // No meta file, do nothing (rm local meta file) + if (pSynch->pmf == NULL) { + if (pLMFile == NULL) { + pSynch->mfChanged = false; + } else { + pSynch->mfChanged = true; + } + tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo)); + return 0; + } + + if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0 || + TSDB_FILE_IS_BAD(pLMFile)) { + // Local has no meta file or has a different meta file, need to copy from remote + pSynch->mfChanged = true; + + if (tsdbSendDecision(pSynch, true) < 0) { + tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo)); + + // Recv from remote + SMFile mf; + SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID}; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + if (tsdbCreateMFile(&mf, false) < 0) { + tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname); + + int32_t readLen = (int32_t)pSynch->pmf->info.size; + int32_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv metafile since %s, ret:%d readLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret, + readLen); + tsdbCloseMFile(&mf); + tsdbRemoveMFile(&mf); + return -1; + } + + tsdbInfo("vgId:%d, metafile is received, size:%d", REPO_ID(pRepo), readLen); + + mf.info = pSynch->pmf->info; + tsdbCloseMFile(&mf); + tsdbUpdateMFile(REPO_FS(pRepo), &mf); + } else { + pSynch->mfChanged = false; + tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo)); + if (tsdbSendDecision(pSynch, false) < 0) { + tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + tsdbUpdateMFile(REPO_FS(pRepo), pLMFile); + } + + return 0; +} + +static int32_t tsdbSendMetaInfo(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen = 0; + SMFile * pMFile = pRepo->fs->cstatus->pmf; + + if (pMFile) { + tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM); + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { + tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + void *ptr = SYNC_BUFFER(pSynch); + taosEncodeFixedU32(&ptr, tlen); + void *tptr = ptr; + if (pMFile) { + tsdbEncodeSMFileEx(&ptr, pMFile); + taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); + } + + int32_t writeLen = tlen + sizeof(uint32_t); + int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret, + writeLen); + return -1; + } + + tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen); + return 0; +} + +static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen = 0; + char buf[64] = {0}; + + int32_t readLen = sizeof(uint32_t); + int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); + return -1; + } + + taosDecodeFixedU32(buf, &tlen); + + tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen); + if (tlen == 0) { + pSynch->pmf = NULL; + return 0; + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { + tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); + if (ret != tlen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen); + return -1; + } + + tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen); + if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { + terrno = TSDB_CODE_TDB_MESSED_MSG; + tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + pSynch->pmf = &(pSynch->mf); + tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf); + + return 0; +} + +static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) { + STsdbRepo *pRepo = pSynch->pRepo; + uint8_t decision = toSend; + + int32_t writeLen = sizeof(uint8_t); + int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); + return -1; + } + + return 0; +} + +static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) { + STsdbRepo *pRepo = pSynch->pRepo; + uint8_t decision = 0; + + int32_t readLen = sizeof(uint8_t); + int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); + return -1; + } + + *toSend = decision; + return 0; +} + +static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + STsdbFS * pfs = REPO_FS(pRepo); + SFSIter fsiter; + SDFileSet *pSet; + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + + do { + pSet = tsdbFSIterNext(&fsiter); + if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) { + tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, + tstrerror(terrno)); + return -1; + } + + // No more file set to send, jut break + if (pSet == NULL) { + tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo)); + break; + } + } while (true); + + return 0; +} + +static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + STsdbFS * pfs = REPO_FS(pRepo); + SFSIter fsiter; + SDFileSet *pLSet; // Local file set + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + + pLSet = tsdbFSIterNext(&fsiter); + if (tsdbRecvDFileSetInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + while (true) { + if (pLSet == NULL && pSynch->pdf == NULL) { + tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo)); + break; + } else { + tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1, + pSynch->pdf != NULL ? pSynch->pdf->fid : -1); + } + + if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) { + // remote not has pLSet->fid set, just remove local (do nothing to remote the fset) + tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid, + pSynch->pdf != NULL ? pSynch->pdf->fid : -1); + pLSet = tsdbFSIterNext(&fsiter); + } else { + if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) && + tsdbFSetIsOk(pLSet)) { + // Just keep local files and notify remote not to send + tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid); + + if (tsdbUpdateDFileSet(pfs, pLSet) < 0) { + tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (tsdbSendDecision(pSynch, false) < 0) { + tsdbError("vgId:%d, filed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } else { + // Need to copy from remote + tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid); + + // Notify remote to send there file here + if (tsdbSendDecision(pSynch, true) < 0) { + tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // Create local files and copy from remote + SDiskID did; + SDFileSet fset; + + tfsAllocDisk(tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)), &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs)); + + // Create new FSET + if (tsdbCreateDFileSet(&fset, false) < 0) { + tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); // local file + SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype); // remote file + + tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo), + pDFile->f.aname, pDFile->info.size, pRDFile->info.size); + + int32_t writeLen = (int32_t)pRDFile->info.size; + int32_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%d writeLen:%d", REPO_ID(pRepo), pDFile->f.aname, + tstrerror(terrno), ret, writeLen); + tsdbCloseDFileSet(&fset); + tsdbRemoveDFileSet(&fset); + return -1; + } + + // Update new file info + pDFile->info = pRDFile->info; + tsdbInfo("vgId:%d, file:%s is received, size:%d", REPO_ID(pRepo), pDFile->f.aname, writeLen); + } + + tsdbCloseDFileSet(&fset); + if (tsdbUpdateDFileSet(pfs, &fset) < 0) { + tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid); + } + + // Move forward + if (tsdbRecvDFileSetInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (pLSet) { + pLSet = tsdbFSIterNext(&fsiter); + } + } + +#if 0 + if (pLSet == NULL) { + // Copy from remote >>>>>>>>>>> + } else { + if (pSynch->pdf == NULL) { + // Remove local file, just ignore ++++++++++++++ + pLSet = tsdbFSIterNext(&fsiter); + } else { + if (pLSet->fid < pSynch->pdf->fid) { + // Remove local file, just ignore ++++++++++++ + pLSet = tsdbFSIterNext(&fsiter); + } else if (pLSet->fid > pSynch->pdf->fid){ + // Copy from remote >>>>>>>>>>>>>> + if (tsdbRecvDFileSetInfo(pSynch) < 0) { + // TODO + return -1; + } + } else { + if (true/*TODO: is same fset*/) { + // No need to copy --------------------- + } else { + // copy from remote >>>>>>>>>>>>>. + } + } + } + } +#endif + } + + return 0; +} + +static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype); + SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype); + + if (memcmp((void *)(TSDB_FILE_INFO(pDFile1)), (void *)(TSDB_FILE_INFO(pDFile2)), sizeof(SDFInfo)) != 0) { + return false; + } + } + + return true; +} + +static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) { + STsdbRepo *pRepo = pSynch->pRepo; + bool toSend = false; + + if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) { + tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno)); + return -1; + } + + // No file any more, no need to send file, just return + if (pSet == NULL) { + return 0; + } + + if (tsdbRecvDecision(pSynch, &toSend) < 0) { + tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid, + tstrerror(terrno)); + return -1; + } + + if (toSend) { + tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid); + + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype); + + if (tsdbOpenDFile(&df, O_RDONLY) < 0) { + tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno)); + return -1; + } + + int32_t writeLen = (int32_t)df.info.size; + tsdbInfo("vgId:%d, file:%s will be sent, size:%d", REPO_ID(pRepo), df.f.aname, writeLen); + + int32_t ret = (int32_t)taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send file:%s since %s, ret:%d writeLen:%d", REPO_ID(pRepo), df.f.aname, + tstrerror(terrno), ret, writeLen); + tsdbCloseDFile(&df); + return -1; + } + + tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname); + tsdbCloseDFile(&df); + } + + tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid); + } else { + tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid); + } + + return 0; +} + +static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen = 0; + + if (pSet) { + tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM); + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { + tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + void *ptr = SYNC_BUFFER(pSynch); + taosEncodeFixedU32(&ptr, tlen); + void *tptr = ptr; + if (pSet) { + tsdbEncodeDFileSetEx(&ptr, pSet); + taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); + } + + int32_t writeLen = tlen + sizeof(uint32_t); + int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); + return -1; + } + + return 0; +} + +static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen; + char buf[64] = {0}; + + int32_t readLen = sizeof(uint32_t); + int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + taosDecodeFixedU32(buf, &tlen); + + tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen); + if (tlen == 0) { + pSynch->pdf = NULL; + return 0; + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { + tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); + if (ret != tlen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { + terrno = TSDB_CODE_TDB_MESSED_MSG; + tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + pSynch->pdf = &(pSynch->df); + tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf); + + return 0; +} + +static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) { + // TODO: may need to stop and restart stream + if (isMfChanged) { + tsdbCloseMeta(pRepo); + tsdbFreeMeta(pRepo->tsdbMeta); + pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo)); + tsdbOpenMeta(pRepo); + tsdbLoadMetaCache(pRepo, true); + } + + tsdbUnRefMemTable(pRepo, pRepo->mem); + tsdbUnRefMemTable(pRepo, pRepo->imem); + pRepo->mem = NULL; + pRepo->imem = NULL; + + if (tsdbRestoreInfo(pRepo) < 0) { + tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index ef5ed6f04459a4213e761f94ad00363ede9ecd26..ac254d6c34ecd152e28bb621348312b938fecd20 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -12,7 +12,7 @@ static double getCurTime() { } typedef struct { - TSDB_REPO_T *pRepo; + STsdbRepo *pRepo; bool isAscend; int tid; uint64_t uid; @@ -143,7 +143,7 @@ TEST(TsdbTest, testInsertSpeed) { // Create and open repository tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1); tsdbCreateRepo(rootDir, &tsdbCfg); - TSDB_REPO_T *repo = tsdbOpenRepo(rootDir, NULL); + STsdbRepo *repo = tsdbOpenRepo(rootDir, NULL); ASSERT_NE(repo, nullptr); // Create table diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 78b9c90979fa74bdd0ce5ecfd53042db30e4e8fa..a7f4f59e07021b659707c59a0c9b2ef916558d52 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -2,10 +2,12 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) -TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) - +TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z rmonotonic) + IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) # ADD_SUBDIRECTORY(tests) @@ -28,5 +30,10 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) TARGET_LINK_LIBRARIES(tutil iconv regex winmm IPHLPAPI ws2_32 wepoll) ELSEIF(TD_DARWIN) + TARGET_LINK_LIBRARIES(tutil m) TARGET_LINK_LIBRARIES(tutil iconv) ENDIF() + +IF (TD_STORAGE) + TARGET_LINK_LIBRARIES(tutil storage) +ENDIF () \ No newline at end of file diff --git a/src/util/inc/buildInfo.h b/src/util/inc/buildInfo.h deleted file mode 100644 index 8d169d618d51a1e0a4191c8c8fbe370536970478..0000000000000000000000000000000000000000 --- a/src/util/inc/buildInfo.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef _TS_BUILD_H_ -#define _TS_BUILD_H_ - -extern const char tsVersion[]; -extern const char tsBuildInfo[]; - -#endif diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index b0319d3e139f72bdaff7993e2e46d61b82c4d0a5..b6b49693f60b15a51eaa35c95555d0f35d36f017 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -32,17 +32,18 @@ typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t keyLen; // length of the key - size_t dataLen; // length of data - int8_t count; // reference count - int8_t removed; // flag to indicate removed + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + int8_t removed; // flag to indicate removed + int8_t count; // reference count char data[]; } SHashNode; #define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) #define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) #define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode)); + typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -82,7 +83,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp * @param pHashObj * @return */ -size_t taosHashGetSize(const SHashObj *pHashObj); +int32_t taosHashGetSize(const SHashObj *pHashObj); /** * put element into hash table, if the element with the same key exists, update it @@ -115,7 +116,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param dsize * @return */ -void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize); +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize); /** * remove item with the specified key @@ -129,16 +130,14 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); +void taosHashEmpty(SHashObj *pHashObj); + /** * clean up hash table * @param handle */ void taosHashCleanup(SHashObj *pHashObj); -/* -void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *); -*/ - /** * * @param pHashObj diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index bf922fe9c44b4f923d44f4a38c1c4c1a7fb20af3..63e62a54c2f679920100b7137cb49377da90789f 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -21,9 +21,11 @@ extern "C" { #endif #include "os.h" +#include "talgo.h" #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) typedef struct SArray { size_t size; @@ -44,9 +46,20 @@ void* taosArrayInit(size_t size, size_t elemSize); * * @param pArray * @param pData + * @param nEles * @return */ -void* taosArrayPush(SArray* pArray, void* pData); +void *taosArrayPushBatch(SArray *pArray, const void *pData, int nEles); + +/** + * + * @param pArray + * @param pData + * @return + */ +static FORCE_INLINE void* taosArrayPush(SArray* pArray, const void* pData) { + return taosArrayPushBatch(pArray, pData, 1); +} /** * @@ -92,6 +105,14 @@ size_t taosArrayGetSize(const SArray* pArray); */ void* taosArrayInsert(SArray* pArray, size_t index, void* pData); +/** + * set data in array + * @param pArray + * @param index + * @param pData + */ +void taosArraySet(SArray* pArray, size_t index, void* pData); + /** * remove data entry of the given index * @param pArray @@ -104,13 +125,13 @@ void taosArrayRemove(SArray* pArray, size_t index); * @param pDst * @param pSrc */ -void taosArrayCopy(SArray* pDst, const SArray* pSrc); +SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize); /** * clone a new array * @param pSrc */ -SArray* taosArrayClone(const SArray* pSrc); +SArray* taosArrayDup(const SArray* pSrc); /** * clear the array (remove all element) @@ -122,7 +143,7 @@ void taosArrayClear(SArray* pArray); * destroy array list * @param pArray */ -void taosArrayDestroy(SArray* pArray); +void* taosArrayDestroy(SArray* pArray); /** * @@ -136,7 +157,7 @@ void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)); * @param pArray * @param compar */ -void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)); +void taosArraySort(SArray* pArray, __compar_fn_t comparFn); /** * sort string array @@ -150,14 +171,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); * @param compar * @param key */ -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn); +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); /** * search the array * @param pArray * @param key */ -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn); +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); #ifdef __cplusplus } diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 33819f6a20ee64ada194d520ef09c6133d4dad96..fdb2595fd8d3b1659800ca3a5b88c32b6fe95e93 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -41,12 +41,15 @@ enum { }; enum { + TAOS_CFG_VTYPE_INT8, TAOS_CFG_VTYPE_INT16, TAOS_CFG_VTYPE_INT32, + TAOS_CFG_VTYPE_UINT16, TAOS_CFG_VTYPE_FLOAT, TAOS_CFG_VTYPE_STRING, TAOS_CFG_VTYPE_IPSTR, TAOS_CFG_VTYPE_DIRECTORY, + TAOS_CFG_VTYPE_DATA_DIRCTORY, }; enum { @@ -78,6 +81,7 @@ extern char * tsCfgStatusStr[]; void taosReadGlobalLogCfg(); bool taosReadGlobalCfg(); void taosPrintGlobalCfg(); +void taosDumpGlobalCfg(); void taosInitConfigOption(SGlobalCfg cfg); SGlobalCfg * taosGetConfigOption(const char *option); diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h deleted file mode 100644 index b2b0ff05f58478e3778d3abab72ae3511f683aca..0000000000000000000000000000000000000000 --- a/src/util/inc/tkvstore.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_KVSTORE_H_ -#define _TD_KVSTORE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#define KVSTORE_FILE_VERSION ((uint32_t)0) - -typedef int (*iterFunc)(void *, void *cont, int contLen); -typedef void (*afterFunc)(void *); - -typedef struct { - int64_t size; // including 512 bytes of header size - int64_t tombSize; - int64_t nRecords; - int64_t nDels; - uint32_t magic; -} SStoreInfo; - -typedef struct { - char * fname; - int fd; - char * fsnap; - int sfd; - char * fnew; - int nfd; - SHashObj * map; - iterFunc iFunc; - afterFunc aFunc; - void * appH; - SStoreInfo info; -} SKVStore; - -#define KVSTORE_MAGIC(s) (s)->info.magic - -int tdCreateKVStore(char *fname); -int tdDestroyKVStore(char *fname); -SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); -void tdCloseKVStore(SKVStore *pStore); -int tdKVStoreStartCommit(SKVStore *pStore); -int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen); -int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid); -int tdKVStoreEndCommit(SKVStore *pStore); -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h index e8380294da47de1c9eb65059263ac820f3d816be..6c96ec0b138cfc229ab4a8bde3f3b374bce49dfd 100644 --- a/src/util/inc/tlist.h +++ b/src/util/inc/tlist.h @@ -47,7 +47,7 @@ typedef struct { #define listNodeFree(n) free(n); SList * tdListNew(int eleSize); -void tdListFree(SList *list); +void * tdListFree(SList *list); void tdListEmpty(SList *list); void tdListPrependNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node); diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h index 426df5cbb28b9c0fcada049c7242730359d2a3fc..b7585bd7155421d1f22e5f989dc7d1ae6f8be491 100644 --- a/src/util/inc/tnettest.h +++ b/src/util/inc/tnettest.h @@ -20,27 +20,7 @@ extern "C" { #endif -typedef struct CmdArguments { - char* host; - char* password; - char* user; - char* auth; - char* database; - char* timezone; - bool is_raw_time; - bool is_use_passwd; - char file[TSDB_FILENAME_LEN]; - char dir[TSDB_FILENAME_LEN]; - int threadNum; - char* commands; - int abort; - int port; - int endPort; - int pktLen; - char* netTestRole; -} CmdArguments; - -void taosNetTest(CmdArguments* args); +void taosNetTest(char *role, char *host, int port, int pkgLen); #ifdef __cplusplus } diff --git a/src/util/inc/tnote.h b/src/util/inc/tnote.h index 552224abf035af6140b46d46ec8bff0e33a15524..7511b61f417c76b312faeb3da0855119658bdd68 100644 --- a/src/util/inc/tnote.h +++ b/src/util/inc/tnote.h @@ -20,41 +20,42 @@ extern "C" { #endif -#include "os.h" -#include "tutil.h" -#include "tglobal.h" - #define MAX_NOTE_LINE_SIZE 66000 #define NOTE_FILE_NAME_LEN 300 - -typedef struct _taosNoteInfo { - int taosNoteFileNum ; - int taosNoteMaxLines; - int taosNoteLines; - char taosNoteName[NOTE_FILE_NAME_LEN]; - int taosNoteFlag; - int taosNoteFd; - int taosNoteOpenInProgress; - pthread_mutex_t taosNoteMutex; -}taosNoteInfo; - -void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...); - -extern taosNoteInfo m_HttpNote; -extern taosNoteInfo m_TscNote; - -extern int tsHttpEnableRecordSql; -extern int tsTscEnableRecordSql; - -#define taosNotePrintHttp(...) \ + +typedef struct { + int32_t fileNum; + int32_t maxLines; + int32_t lines; + int32_t flag; + int32_t fd; + int32_t openInProgress; + char name[NOTE_FILE_NAME_LEN]; + pthread_mutex_t mutex; +} SNoteObj; + +extern SNoteObj tsHttpNote; +extern SNoteObj tsTscNote; +extern SNoteObj tsInfoNote; + +void taosInitNotes(); +void taosNotePrint(SNoteObj* pNote, const char* const format, ...); +void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len); + +#define nPrintHttp(...) \ if (tsHttpEnableRecordSql) { \ - taosNotePrint(&m_HttpNote, __VA_ARGS__); \ + taosNotePrint(&tsHttpNote, __VA_ARGS__); \ + } + +#define nPrintTsc(...) \ + if (tsTscEnableRecordSql) { \ + taosNotePrint(&tsTscNote, __VA_ARGS__); \ + } + +#define nInfo(buffer, len) \ + if (tscEmbedded == 1) { \ + taosNotePrintBuffer(&tsInfoNote, buffer, len); \ } - -#define taosNotePrintTsc(...) \ - if (tsTscEnableRecordSql) { \ - taosNotePrint(&m_TscNote, __VA_ARGS__); \ - } #ifdef __cplusplus } diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index cd5092f30a290c51de49e38b0226bbed637dc0e6..085c10c55198fc92bfacb0628c38f05da508bfb9 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -52,6 +52,8 @@ void *taosIterateRef(int rsetId, int64_t rid); // return the number of references in system int taosListRef(); +#define RID_VALID(x) ((x) > 0) + /* sample code to iterate the refs void demoIterateRefs(int rsetId) { diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index 391cc44accadd785fb46e02aa79cde7617834c70..35b591b61e007dcf6d4edc39810a60454f37c07a 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -20,11 +20,15 @@ extern "C" { #endif +#ifdef WINDOWS +#include "wepoll.h" +#endif + int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); +int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); int32_t taosSetNonblocking(SOCKET sock, int32_t on); SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); @@ -33,7 +37,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(SOCKET sockFd); int32_t taosGetFqdn(char *); -uint32_t taosGetIpFromFqdn(const char *); +uint32_t taosGetIpv4FromFqdn(const char *); void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 7aeb2da6b658f4b8fd1cc7e03583a3cc676ea237..7af03d96af4eb132288f7b8841bc26eedcf545da 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -27,11 +27,14 @@ extern "C" { #define TSQL_TBNAME "TBNAME" #define TSQL_TBNAME_L "tbname" +#define TSQL_BLOCK_DIST "_BLOCK_DIST" +#define TSQL_BLOCK_DIST_L "_block_dist" + // used to denote the minimum unite in sql parsing typedef struct SStrToken { uint32_t n; uint32_t type; - char * z; + char *z; } SStrToken; /** @@ -76,7 +79,7 @@ bool isKeyWord(const char *z, int32_t len); * @param pToken * @return token type, if it is not a number, TK_ILLEGAL will return */ -static FORCE_INLINE int32_t isValidNumber(const SStrToken* pToken) { +static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) { const char* z = pToken->z; int32_t type = TK_ILLEGAL; @@ -111,7 +114,6 @@ static FORCE_INLINE int32_t isValidNumber(const SStrToken* pToken) { type = TK_FLOAT; goto _end; - break; } case '0': { diff --git a/src/util/inc/tworker.h b/src/util/inc/tworker.h new file mode 100644 index 0000000000000000000000000000000000000000..7bc1eba2fdcca0409737d9305911cd2987afe1ab --- /dev/null +++ b/src/util/inc/tworker.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TWORKER_H +#define TDENGINE_TWORKER_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *(*FWorkerThread)(void *pWorker); +struct SWorkerPool; + +typedef struct { + pthread_t thread; // thread + int32_t id; // worker ID + struct SWorkerPool *pPool; +} SWorker; + +typedef struct SWorkerPool { + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + void * qset; + char * name; + SWorker *worker; + FWorkerThread workerFp; + pthread_mutex_t mutex; +} SWorkerPool; + +int32_t tWorkerInit(SWorkerPool *pPool); +void tWorkerCleanup(SWorkerPool *pPool); +void * tWorkerAllocQueue(SWorkerPool *pPool, void *ahandle); +void tWorkerFreeQueue(SWorkerPool *pPool, void *pQueue); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 0e3e0d3e2437f4595179c880a37642e047eb4331..ede3e1e4ce3d1c9e5cadbd6bc21c009f0c5f5ca3 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -189,7 +189,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp return pHashObj; } -size_t taosHashGetSize(const SHashObj *pHashObj) { return (pHashObj == NULL) ? 0 : pHashObj->size; } +int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); @@ -271,10 +271,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetCB(pHashObj, key, keyLen, NULL, NULL, 0); + return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL, 0); } -void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { return NULL; } @@ -313,10 +313,10 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f } if (d != NULL) { - memcpy(d, GET_HASH_NODE_DATA(pNode), dsize); - } else { - data = GET_HASH_NODE_DATA(pNode); + memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); } + + data = GET_HASH_NODE_DATA(pNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -472,7 +472,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi return 0; } -void taosHashCleanup(SHashObj *pHashObj) { +void taosHashEmpty(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } @@ -481,29 +481,38 @@ void taosHashCleanup(SHashObj *pHashObj) { __wr_lock(&pHashObj->lock, pHashObj->type); - if (pHashObj->hashList) { - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - assert(pEntry->next == 0); - continue; - } + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + assert(pEntry->next == 0); + continue; + } - pNode = pEntry->next; - assert(pNode != NULL); + pNode = pEntry->next; + assert(pNode != NULL); - while (pNode) { - pNext = pNode->next; - FREE_HASH_NODE(pHashObj, pNode); + while (pNode) { + pNext = pNode->next; + FREE_HASH_NODE(pHashObj, pNode); - pNode = pNext; - } + pNode = pNext; } - free(pHashObj->hashList); + pEntry->num = 0; + pEntry->next = NULL; } + pHashObj->size = 0; __wr_unlock(&pHashObj->lock, pHashObj->type); +} + +void taosHashCleanup(SHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + taosHashEmpty(pHashObj); + tfree(pHashObj->hashList); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); @@ -654,7 +663,7 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; - pNewNode->dataLen = dsize; + pNewNode->dataLen = (uint32_t) dsize; pNewNode->count = 1; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index bec2fac7dfd03ada681b9011df7e465cd0a49f0c..4dde5dbba24adfcda0fe794f2f36d6c059354f55 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -55,24 +55,29 @@ static int32_t taosArrayResize(SArray* pArray) { return 0; } -void* taosArrayPush(SArray* pArray, void* pData) { +void* taosArrayPushBatch(SArray* pArray, const void* pData, int nEles) { if (pArray == NULL || pData == NULL) { return NULL; } - if (pArray->size >= pArray->capacity) { - int32_t ret = taosArrayResize(pArray); - - // failed to push data into buffer due to the failure of memory allocation - if (ret != 0) { + if (pArray->size + nEles > pArray->capacity) { + size_t tsize = (pArray->capacity << 1u); + while (pArray->size + nEles > tsize) { + tsize = (tsize << 1u); + } + + pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); + if (pArray->pData == NULL) { return NULL; } + + pArray->capacity = tsize; } void* dst = TARRAY_GET_ELEM(pArray, pArray->size); - memcpy(dst, pData, pArray->elemSize); + memcpy(dst, pData, pArray->elemSize * nEles); - pArray->size += 1; + pArray->size += nEles; return dst; } @@ -133,6 +138,11 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { return dst; } +void taosArraySet(SArray* pArray, size_t index, void* pData) { + assert(index < pArray->size); + memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize); +} + void taosArrayRemove(SArray* pArray, size_t index) { assert(index < pArray->size); @@ -146,26 +156,17 @@ void taosArrayRemove(SArray* pArray, size_t index) { pArray->size -= 1; } -void taosArrayCopy(SArray* pDst, const SArray* pSrc) { - assert(pSrc != NULL && pDst != NULL); - - if (pDst->capacity < pSrc->size) { - void* pData = realloc(pDst->pData, pSrc->size * pSrc->elemSize); - if (pData == NULL) { // todo handle oom - - } else { - pDst->pData = pData; - pDst->capacity = pSrc->size; - } - } - - memcpy(pDst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); - pDst->elemSize = pSrc->elemSize; - pDst->capacity = pSrc->size; - pDst->size = pSrc->size; +SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { + assert(src != NULL && elemSize > 0); + SArray* pDst = taosArrayInit(size, elemSize); + + memcpy(pDst->pData, src, elemSize * size); + pDst->size = size; + + return pDst; } -SArray* taosArrayClone(const SArray* pSrc) { +SArray* taosArrayDup(const SArray* pSrc) { assert(pSrc != NULL); if (pSrc->size == 0) { // empty array list @@ -184,13 +185,13 @@ void taosArrayClear(SArray* pArray) { pArray->size = 0; } -void taosArrayDestroy(SArray* pArray) { - if (pArray == NULL) { - return; +void* taosArrayDestroy(SArray* pArray) { + if (pArray) { + free(pArray->pData); + free(pArray); } - free(pArray->pData); - free(pArray); + return NULL; } void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { @@ -210,18 +211,18 @@ void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { taosArrayDestroy(pArray); } -void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) { +void taosArraySort(SArray* pArray, __compar_fn_t compar) { assert(pArray != NULL); assert(compar != NULL); qsort(pArray->pData, pArray->size, pArray->elemSize, compar); } -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn) { +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { assert(pArray != NULL && comparFn != NULL); assert(key != NULL); - return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn); + return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); } void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { @@ -229,11 +230,11 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); } -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn) { +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags) { assert(pArray != NULL); assert(key != NULL); - void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn); + void* p = taosbsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); if (p == NULL) { return NULL; } diff --git a/src/util/src/tbuffer.c b/src/util/src/tbuffer.c index 240f744ea3083e1b21f60076032483f828298167..a2cb32c1f45b1930036e545a5c1508cff658e0d9 100644 --- a/src/util/src/tbuffer.c +++ b/src/util/src/tbuffer.c @@ -191,7 +191,8 @@ double tbufReadDouble(SBufferReader* buf) { // writer functions void tbufCloseWriter( SBufferWriter* buf ) { - (*buf->allocator)( buf->data, 0 ); + tfree(buf->data); +// (*buf->allocator)( buf->data, 0 ); // potential memory leak. buf->data = NULL; buf->pos = 0; buf->size = 0; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2571f11ba41b653e1fcc639cf1c22ef1f50b3448..c0cc8ce33942d1e6784db0f7f4b86757f8f7f593 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -278,7 +278,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen } SCacheDataNode* ptNode = NULL; - taosHashGetCB(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode, sizeof(void*)); + taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode, sizeof(void*)); void* pData = (ptNode != NULL)? ptNode->data:NULL; @@ -510,7 +510,9 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { } pCacheObj->deleting = 1; - pthread_join(pCacheObj->refreshWorker, NULL); + if (taosCheckPthreadValid(pCacheObj->refreshWorker)) { + pthread_join(pCacheObj->refreshWorker, NULL); + } uInfo("cache:%s will be cleaned up", pCacheObj->name); doCleanupDataCache(pCacheObj); @@ -547,13 +549,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { return; } + __cache_wr_lock(pCacheObj); STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; - pElem->prev = NULL; + pElem->prev = NULL; + pElem->next = NULL; pNode->inTrashcan = true; pNode->pTNodeHeader = pElem; - __cache_wr_lock(pCacheObj); pElem->next = pCacheObj->pTrash; if (pCacheObj->pTrash) { pCacheObj->pTrash->prev = pElem; @@ -563,8 +566,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->numOfElemsInTrash++; __cache_unlock(pCacheObj); - uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, - pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash); + uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, + pNode->data, pElem, pCacheObj->numOfElemsInTrash); } void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index ff67b1f3ecf88c6da40d7b47cdac426a51636a48..4d18ef14e2a1985259b11ff65391616c9d4706b2 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,4 +1,4 @@ -#include "taosdef.h" +#include "ttype.h" #include "tcompare.h" #include "tarray.h" @@ -30,28 +30,51 @@ int32_t compareInt8Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { - int64_t lhs = GET_INT64_VAL(pLeft); - double rhs = GET_DOUBLE_VAL(pRight); - if (fabs(lhs - rhs) < FLT_EPSILON) { - return 0; - } else { - return (lhs > rhs) ? 1 : -1; - } +int32_t compareUint32Val(const void *pLeft, const void *pRight) { + int32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } -int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { - double lhs = GET_DOUBLE_VAL(pLeft); - int64_t rhs = GET_INT64_VAL(pRight); - if (fabs(lhs - rhs) < FLT_EPSILON) { - return 0; - } else { - return (lhs > rhs) ? 1 : -1; - } +int32_t compareUint64Val(const void *pLeft, const void *pRight) { + int64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareUint16Val(const void *pLeft, const void *pRight) { + int16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareUint8Val(const void* pLeft, const void* pRight) { + uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareFloatVal(const void *pLeft, const void *pRight) { - float ret = GET_FLOAT_VAL(pLeft) - GET_FLOAT_VAL(pRight); + float p1 = GET_FLOAT_VAL(pLeft); + float p2 = GET_FLOAT_VAL(pRight); + + if (isnan(p1) && isnan(p2)) { + return 0; + } + + if (isnan(p1)) { + return -1; + } + + if (isnan(p2)) { + return 1; + } + + float ret = p1 - p2; if (fabs(ret) < FLT_EPSILON) { return 0; } else { @@ -60,7 +83,22 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) { } int32_t compareDoubleVal(const void *pLeft, const void *pRight) { - double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); + double p1 = GET_DOUBLE_VAL(pLeft); + double p2 = GET_DOUBLE_VAL(pRight); + + if (isnan(p1) && isnan(p2)) { + return 0; + } + + if (isnan(p1)) { + return -1; + } + + if (isnan(p2)) { + return 1; + } + + double ret = p1 - p2; if (fabs(ret) < FLT_EPSILON) { return 0; } else { @@ -239,7 +277,7 @@ int32_t taosArrayCompareString(const void* a, const void* b) { static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { const SArray* arr = (const SArray*) pRight; - return taosArraySearchString(arr, pLeft, taosArrayCompareString) == NULL ? 0 : 1; + return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1; } static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { @@ -259,32 +297,14 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; switch (type) { - case TSDB_DATA_TYPE_SMALLINT: { - comparFn = compareInt16Val; break; - } - - case TSDB_DATA_TYPE_INT: { - comparFn = compareInt32Val; break; - } - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: { - comparFn = compareInt64Val; break; - } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT:{ - comparFn = compareInt8Val; break; - } - - case TSDB_DATA_TYPE_FLOAT: { - comparFn = compareFloatVal; break; - } - - case TSDB_DATA_TYPE_DOUBLE: { - comparFn = compareDoubleVal; break; - } - + case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; + case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; case TSDB_DATA_TYPE_BINARY: { if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ comparFn = compareStrPatternComp; @@ -303,10 +323,14 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } else { comparFn = compareLenPrefixedWStr; } - break; } - + + case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; + case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; + default: comparFn = compareInt32Val; break; @@ -339,15 +363,24 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; - + case TSDB_DATA_TYPE_UTINYINT: + comparFn = compareUint8Val; + break; + case TSDB_DATA_TYPE_USMALLINT: + comparFn = compareUint16Val; + break; + case TSDB_DATA_TYPE_UINT: + comparFn = compareUint32Val; + break; + case TSDB_DATA_TYPE_UBIGINT: + comparFn = compareUint64Val; + break; case TSDB_DATA_TYPE_BINARY: comparFn = compareLenPrefixedStr; break; - case TSDB_DATA_TYPE_NCHAR: comparFn = compareLenPrefixedWStr; break; - default: comparFn = compareInt32Val; break; @@ -359,8 +392,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { switch (type) { case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); + case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); + case TSDB_DATA_TYPE_FLOAT: DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); case TSDB_DATA_TYPE_TINYINT: diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 173de294cf333e20b9649a5634905421786a1b30..c4bd57760222ac3da7d25510cc2f434fe0cf0cac 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -20,7 +20,7 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" -#include "tsystem.h" +#include "tsocket.h" #include "tutil.h" SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; @@ -95,32 +95,73 @@ static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { } } -static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { +static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { + int32_t value = atoi(input_value); + uint16_t *option = (uint16_t *)cfg->ptr; + if (value < cfg->minValue || value > cfg->maxValue) { + uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", + cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); + } else { + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { + *option = (uint16_t)value; + cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; + } else { + uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, + tsCfgStatusStr[cfg->cfgStatus], *option); + } + } +} + +static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { + int32_t value = atoi(input_value); + int8_t *option = (int8_t *)cfg->ptr; + if (value < cfg->minValue || value > cfg->maxValue) { + uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", + cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); + } else { + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { + *option = (int8_t)value; + cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; + } else { + uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, + tsCfgStatusStr[cfg->cfgStatus], *option); + } + } +} + +static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { int length = (int)strlen(input_value); char *option = (char *)cfg->ptr; if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", - cfg->option, input_value, cfg->ptrLength, option); + uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option, + input_value, cfg->ptrLength, option); + return false; } else { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { wordexp_t full_path; if (0 != wordexp(input_value, &full_path, 0)) { printf("\nconfig dir: %s wordexp fail! reason:%s\n", input_value, strerror(errno)); wordfree(&full_path); - return; + return false; } - + if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { strcpy(option, full_path.we_wordv[0]); } - + wordfree(&full_path); + char tmp[1025] = {0}; + if (realpath(option, tmp) != NULL) { + strcpy(option, tmp); + } + int code = taosMkDir(option, 0755); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("config option:%s, input value:%s, directory not exist, create fail:%s", - cfg->option, input_value, strerror(errno)); + uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, + strerror(errno)); + return false; } cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; } else { @@ -128,6 +169,8 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { tsCfgStatusStr[cfg->cfgStatus], option); } } + + return true; } static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { @@ -197,19 +240,25 @@ SGlobalCfg *taosGetConfigOption(const char *option) { return NULL; } -static void taosReadConfigOption(const char *option, char *value) { +static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) { for (int i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; if (strcasecmp(cfg->option, option) != 0) continue; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + taosReadInt8Config(cfg, value); + break; case TAOS_CFG_VTYPE_INT16: taosReadInt16Config(cfg, value); break; case TAOS_CFG_VTYPE_INT32: taosReadInt32Config(cfg, value); break; + case TAOS_CFG_VTYPE_UINT16: + taosReadUInt16Config(cfg, value); + break; case TAOS_CFG_VTYPE_FLOAT: taosReadFloatConfig(cfg, value); break; @@ -222,6 +271,11 @@ static void taosReadConfigOption(const char *option, char *value) { case TAOS_CFG_VTYPE_DIRECTORY: taosReadDirectoryConfig(cfg, value); break; + case TAOS_CFG_VTYPE_DATA_DIRCTORY: + if (taosReadDirectoryConfig(cfg, value)) { + taosReadDataDirCfg(value, value2, value3); + } + break; default: uError("config option:%s, input value:%s, can't be recognized", option, value); break; @@ -240,9 +294,6 @@ void taosReadGlobalLogCfg() { int olen, vlen; char fileName[PATH_MAX] = {0}; - mDebugFlag = 135; - sdbDebugFlag = 135; - wordexp_t full_path; if ( 0 != wordexp(configDir, &full_path, 0)) { printf("\nconfig file: %s wordexp fail! reason:%s\n", configDir, strerror(errno)); @@ -305,8 +356,8 @@ void taosReadGlobalLogCfg() { } bool taosReadGlobalCfg() { - char * line, *option, *value, *value1; - int olen, vlen, vlen1; + char * line, *option, *value, *value2, *value3; + int olen, vlen, vlen2, vlen3; char fileName[PATH_MAX] = {0}; sprintf(fileName, "%s/taos.cfg", configDir); @@ -329,8 +380,8 @@ bool taosReadGlobalCfg() { while (!feof(fp)) { memset(line, 0, len); - option = value = NULL; - olen = vlen = 0; + option = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; tgetline(&line, &len, fp); line[len - 1] = 0; @@ -343,11 +394,14 @@ bool taosReadGlobalCfg() { if (vlen == 0) continue; value[vlen] = 0; - // For dataDir, the format is: - // dataDir /mnt/disk1 0 - paGetToken(value + vlen + 1, &value1, &vlen1); - - taosReadConfigOption(option, value); + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + taosReadConfigOption(option, value, value2, value3); } fclose(fp); @@ -379,12 +433,18 @@ void taosPrintGlobalCfg() { blank[blankLen] = 0; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_INT16: uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; case TAOS_CFG_VTYPE_INT32: uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; + case TAOS_CFG_VTYPE_UINT16: + uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_FLOAT: uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; @@ -399,4 +459,66 @@ void taosPrintGlobalCfg() { } taosPrintOsInfo(); + taosPrintDataDirCfg(); + uInfo("=================================="); +} + +static void taosDumpCfg(SGlobalCfg *cfg) { + int optionLen = (int)strlen(cfg->option); + int blankLen = TSDB_CFG_PRINT_LEN - optionLen; + blankLen = blankLen < 0 ? 0 : blankLen; + + char blank[TSDB_CFG_PRINT_LEN]; + memset(blank, ' ', TSDB_CFG_PRINT_LEN); + blank[blankLen] = 0; + + switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_INT16: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_INT32: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_UINT16: + printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_FLOAT: + printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; + case TAOS_CFG_VTYPE_STRING: + case TAOS_CFG_VTYPE_IPSTR: + case TAOS_CFG_VTYPE_DIRECTORY: + printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); + break; + default: + break; + } +} + +void taosDumpGlobalCfg() { + printf("taos global config:\n"); + printf("==================================\n"); + for (int i = 0; i < tsGlobalConfigNum; ++i) { + SGlobalCfg *cfg = tsGlobalConfig + i; + if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; + if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; + + taosDumpCfg(cfg); + } + + printf("\ntaos local config:\n"); + printf("==================================\n"); + + for (int i = 0; i < tsGlobalConfigNum; ++i) { + SGlobalCfg *cfg = tsGlobalConfig + i; + if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; + + taosDumpCfg(cfg); + } } diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 1eb6893cbc639be013104fe636f6af373a61b3c3..4a011b7cc7407acabe249b577280f7bbda58f79d 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -24,17 +24,416 @@ typedef struct { int32_t val; const char* str; } STaosError; - #include "os.h" #include "taoserror.h" - static threadlocal int32_t tsErrno; int32_t* taosGetErrno() { return &tsErrno; } +#ifdef TAOS_ERROR_C +#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str=(msg)}, +#else +#define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = TAOS_DEF_ERROR_CODE(mod, code); +#endif + +#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) +#define TAOS_SUCCEEDED(err) ((err) >= 0) +#define TAOS_FAILED(err) ((err) < 0) + +#ifdef TAOS_ERROR_C +STaosError errors[] = { + {.val = 0, .str = "success"}, +#endif + +// rpc +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_REQUIRED, "Authentication required") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Redirect") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NOT_READY, "System not ready") // peer is not ready to process data +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ALREADY_PROCESSED, "Message already processed") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED, "Last session not finished") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MISMATCHED_LINK_ID, "Mismatched meter id") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TOO_SLOW, "Processing of request timed out") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, "Number of sessions reached limit") // too many sessions +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_APP_ERROR, "Unexpected generic error in RPC") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_UNEXPECTED_RESPONSE, "Unexpected response") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VALUE, "Invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID, "Invalid transaction id") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, "Invalid session id") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, "Invalid message type") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, "Invalid response type") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, "Client and server's time is not synchronized") +TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, "Database not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION, "Invalid app version") + +//common & util +TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, "Operation not supported") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, "Memory corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, "Out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, "Invalid config message") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, "Data file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, "Invalid Ref ID") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, "Ref is already there") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") + +//client +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, "Invalid SQL statement") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE, "Invalid qhandle") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TIME_STAMP, "Invalid combination of client/service time") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VALUE, "Invalid value in client") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VERSION, "Invalid client version") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_IE, "Invalid client ie") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_FQDN, "Invalid host name") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_USER_LENGTH, "Invalid user name") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH, "Invalid password") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, "Database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, "Table name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, "Invalid connection") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, "System out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_DISKSPACE, "System out of disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, "Query cache erased") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, "Query terminated") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, "Result set too large to be sorted") // too many result for ordered super table projection query +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, "Application error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, "Disconnected from service") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, "No write permission") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, "Connection killed") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, "Syntax error in SQL") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, "Database not specified or available") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, "Table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too long, check maxSQLLength config") + +// mnode +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Messag need to be reprocessed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Unexpected generic error in mnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION, "Incompatible protocol version") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN, "Invalid message length") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE, "Invalid message type") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS, "Too many connections") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_OUT_OF_MEMORY, "Out of memory in mnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, "Invalid stream id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, "Invalid connection id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_IS_RUNNING, "mnode is alreay running") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC, "failed to config sync") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_START_SYNC, "failed to start sync") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CREATE_DIR, "failed to create mnode dir") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_INIT_STEP, "failed to init components") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, "Object already there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, "Unexpected generic error in sdb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE, "Invalid table type") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE, "Object not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW, "Invalid meta row") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE, "Invalid key type") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "DNode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "DNode does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, "Master DNode cannot be removed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of DNodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, "Cluster cfg inconsistent") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, "Invalid dnode cfg option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, "Balance already enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, "Dnode not avaliable") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED, "Dnode Id not configured") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED, "Dnode Ep not configured") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, "Invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, "Invalid account options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, "Account authorization has expired") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, "User already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, "Invalid user") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT, "Invalid user format") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, "Invalid password format") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from conn") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST, "Table already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") // operation only available for super table +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, "Cannot delete monitor database") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database not available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, "Database unsynced") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, "Invalid database option: days out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, "Invalid database option: keep >= keep1 >= keep0 >= days") + +// dnode +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, "Dnode out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, "No permission for disk files in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, "Too many vnode directories") + +// vnode +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, "Action need to be reprocessed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Invalid Vgroup ID") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED, "Vnode initialization failed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISKSPACE, "System out of disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS, "No write permission for disk files") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, "Missing data file") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, "Out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, "Unexpected generic error in vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, "Invalid version file") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, "Database memory is full for commit failed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, "Database memory is full for waiting commit") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_DROPPING, "Database is dropping") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_BALANCING, "Database is balancing") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, "Database suspended") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing") + +// tsdb +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, "Invalid table type") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, "Invalid table schema version") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, "No diskspace for tsdb") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, "No permission for disk files") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, "Data file(s) corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, "Out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, "Tag too old") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, "Timestamp data out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, "Submit message is messed up") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, "Invalid operation") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, "Invalid creation of table") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, "No table data in memory skiplist") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, "File already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, "Need to reconfigure table") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, "Invalid information to create table") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") + +// query +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, "Invalid message") // failed to validate the sql expression msg by vnode +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE, "No diskspace for query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, "System out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, "Unexpected generic error in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, "Duplicated join key") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") + + +// grant +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, "DNode creation limited by licence") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, "Account creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, "Table creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, "DB creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, "User creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, "Conn creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, "Stream creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, "Write speed limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, "Storage capacity limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, "Query time limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, "CPU cores limited by license") + +// sync +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, "Invalid Sync Configuration") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, "Sync module not enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, "Invalid Sync version") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, "Sync confirm expired") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TOO_MANY_FWDINFO, "Too many sync fwd infos") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_PROTOCOL, "Mismatched protocol") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_CLUSTERID, "Mismatched clusterId") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_SIGNATURE, "Mismatched signature") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CHECKSUM, "Invalid msg checksum") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGLEN, "Invalid msg length") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, "Invalid msg type") + +// wal +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Unexpected generic error in wal") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") + +// http +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, "http server is not onlin") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, "url is not support") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_URL, "invalid url format") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, "no enough memory") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUSET_TOO_BIG, "request size is too big") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_AUTH_INFO, "no auth info input") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_MSG_INPUT, "request is empty") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_SQL_INPUT, "no sql input") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_EXEC_USEDB, "no need to execute use db cmd") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SESSION_FULL, "session list was full") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR, "generate taosd token error") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_MULTI_REQUEST, "size of multi request is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, "failed to create gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, "failed to finish gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_LOGIN_FAILED, "failed to login") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, "invalid http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, "invalid content length") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, "invalid type of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, "invalid format of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, "invalid basic Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, "invalid taosd Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, "failed to parse method") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, "failed to parse target") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, "failed to parse http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, "failed to parse sp") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, "failed to parse status") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, "failed to parse phrase") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, "failed to parse crlf") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, "failed to parse header") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, "failed to parse header key") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, "failed to parse header val") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, "failed to parse chunk size") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, "failed to parse chunk") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, "failed to parse end section") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, "invalid parse state") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, "failed to parse error section") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_NULL, "query size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_SIZE, "query size can not more than 100") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR, "parse grafana json error") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_NOT_INPUT, "database name can not be null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_TOO_LONG, "database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_INVALID_JSON, "invalid telegraf json fromat") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_NULL, "metrics size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_SIZE, "metrics size can not more than 1K") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NULL, "metric name not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_TYPE, "metric name type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_NULL, "metric name length is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_LONG, "metric name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, "timestamp not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, "timestamp type should be integer") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, "timestamp value smaller than 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, "tags not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, "tags size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, "tag is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, "tag name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, "tag name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE, "tag value type should be number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_NULL, "tag value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_NULL, "table is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_SIZE, "table name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_NULL, "fields not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_0, "fields size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG, "fields size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NULL, "field is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_NULL, "field name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE, "field name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE, "field value type should be number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL, "field value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_HOST_NOT_STRING, "host type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST, "stable not exist") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_NOT_INPUT, "database name can not be null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_TOO_LONG, "database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_INVALID_JSON, "invalid opentsdb json fromat") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_NULL, "metrics size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_SIZE, "metrics size can not more than 10K") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NULL, "metric name not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_TYPE, "metric name type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_NULL, "metric name length is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_LONG, "metric name length can not more than 22") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_NULL, "timestamp not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE, "timestamp type should be integer") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL, "timestamp value smaller than 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_NULL, "tags not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_0, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG, "tags size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NULL, "tag is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_NULL, "tag name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_SIZE, "tag name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, "tag value type should be boolean, number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, "tag value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, "tag value can not more than 64") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, "value not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, "value type should be boolean, number or string") + +// odbc +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, "out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, "convertion not a valid literal input") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, "convertion undefined") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC_FRAC, "convertion fractional truncated") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC, "convertion truncated") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_SUPPORT, "convertion not supported") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_OOR, "convertion numeric value out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OUT_OF_RANGE, "out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NOT_SUPPORT, "not supported yet") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_HANDLE, "invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_RESULT, "no result set") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_FIELDS, "no fields returned") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_CURSOR, "invalid cursor") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_STATEMENT_NOT_READY, "statement not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONNECTION_BUSY, "connection still busy") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_CONNSTR, "bad connection string") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_ARG, "bad argument") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_VALID_TS, "not a valid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE, "src too large") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, "src bad sequence") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, "src incomplete") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, "src general") + +// tfs +TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY, "tfs out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT, "tfs too many mount") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY, "tfs duplicate primary mount") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_PRIMARY_DISK, "tfs no primary mount") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER, "tfs no mount at tier") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") + +#ifdef TAOS_ERROR_C +}; +#endif static int tsCompareTaosError(const void* a, const void* b) { const STaosError* x = (const STaosError*)a; diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 64fea2843b6d54488c77f9da6aa4965749855229..dd621b3199189bc30f851c9a9289e4b10f6510f3 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -54,12 +54,12 @@ static int64_t tfOpenImp(int32_t fd) { } int64_t tfOpen(const char *pathname, int32_t flags) { - int32_t fd = open(pathname, flags); + int32_t fd = open(pathname, flags | O_BINARY); return tfOpenImp(fd); } int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = open(pathname, flags, mode); + int32_t fd = open(pathname, flags | O_BINARY, mode); return tfOpenImp(fd); } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c deleted file mode 100644 index 2b1d13c78b3d52f511e0c01e5472a2cd19c5255b..0000000000000000000000000000000000000000 --- a/src/util/src/tkvstore.c +++ /dev/null @@ -1,621 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#define TAOS_RANDOM_FILE_FAIL_TEST -#include "os.h" -#include "hash.h" -#include "taoserror.h" -#include "tchecksum.h" -#include "tcoding.h" -#include "tkvstore.h" -#include "tulog.h" - -#define TD_KVSTORE_HEADER_SIZE 512 -#define TD_KVSTORE_MAJOR_VERSION 1 -#define TD_KVSTORE_MAINOR_VERSION 0 -#define TD_KVSTORE_SNAP_SUFFIX ".snap" -#define TD_KVSTORE_NEW_SUFFIX ".new" -#define TD_KVSTORE_INIT_MAGIC 0xFFFFFFFF - -typedef struct { - uint64_t uid; - int64_t offset; - int64_t size; -} SKVRecord; - -static int tdInitKVStoreHeader(int fd, char *fname); -static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo); -static void * tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo); -static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); -static char * tdGetKVStoreSnapshotFname(char *fdata); -static char * tdGetKVStoreNewFname(char *fdata); -static void tdFreeKVStore(SKVStore *pStore); -static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo); -static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version); -static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord); -static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord); -static int tdRestoreKVStore(SKVStore *pStore); - -int tdCreateKVStore(char *fname) { - int fd = open(fname, O_RDWR | O_CREAT, 0755); - if (fd < 0) { - uError("failed to open file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (tdInitKVStoreHeader(fd, fname) < 0) goto _err; - - if (fsync(fd) < 0) { - uError("failed to fsync file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (close(fd) < 0) { - uError("failed to close file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - return 0; - -_err: - if (fd >= 0) close(fd); - (void)remove(fname); - return -1; -} - -int tdDestroyKVStore(char *fname) { - if (remove(fname) < 0) { - uError("failed to remove file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) { - SStoreInfo info = {0}; - uint32_t version = 0; - - SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH); - if (pStore == NULL) return NULL; - - pStore->fd = open(pStore->fname, O_RDWR); - if (pStore->fd < 0) { - uError("failed to open file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pStore->sfd = open(pStore->fsnap, O_RDONLY); - if (pStore->sfd < 0) { - if (errno != ENOENT) { - uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - } else { - uDebug("file %s exists, try to recover the KV store", pStore->fsnap); - if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info, &version) < 0) { - if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err; - } else { - if (version != KVSTORE_FILE_VERSION) { - uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fsnap, - version, KVSTORE_FILE_VERSION); - } - - if (taosFtruncate(pStore->fd, info.size) < 0) { - uError("failed to truncate %s to %" PRId64 " size since %s", pStore->fname, info.size, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; - if (fsync(pStore->fd) < 0) { - uError("failed to fsync file %s since %s", pStore->fname, strerror(errno)); - goto _err; - } - } - - close(pStore->sfd); - pStore->sfd = -1; - (void)remove(pStore->fsnap); - } - - if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info, &version) < 0) goto _err; - if (version != KVSTORE_FILE_VERSION) { - uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fname, version, - KVSTORE_FILE_VERSION); - } - - pStore->info.size = TD_KVSTORE_HEADER_SIZE; - pStore->info.magic = info.magic; - - if (tdRestoreKVStore(pStore) < 0) goto _err; - - close(pStore->fd); - pStore->fd = -1; - - return pStore; - -_err: - if (pStore->fd > 0) { - close(pStore->fd); - pStore->fd = -1; - } - if (pStore->sfd > 0) { - close(pStore->sfd); - pStore->sfd = -1; - } - tdFreeKVStore(pStore); - return NULL; -} - -void tdCloseKVStore(SKVStore *pStore) { tdFreeKVStore(pStore); } - -int tdKVStoreStartCommit(SKVStore *pStore) { - ASSERT(pStore->fd < 0); - - pStore->fd = open(pStore->fname, O_RDWR); - if (pStore->fd < 0) { - uError("failed to open file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT, 0755); - if (pStore->sfd < 0) { - uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { - uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (fsync(pStore->sfd) < 0) { - uError("failed to fsync file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (close(pStore->sfd) < 0) { - uError("failed to close file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - pStore->sfd = -1; - - if (lseek(pStore->fd, 0, SEEK_END) < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(pStore->info.size == lseek(pStore->fd, 0, SEEK_CUR)); - - return 0; - -_err: - if (pStore->sfd > 0) { - close(pStore->sfd); - pStore->sfd = -1; - (void)remove(pStore->fsnap); - } - if (pStore->fd > 0) { - close(pStore->fd); - pStore->fd = -1; - } - return -1; -} - -int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) { - SKVRecord rInfo = {0}; - char buf[64] = "\0"; - char * pBuf = buf; - - rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR); - if (rInfo.offset < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - rInfo.uid = uid; - rInfo.size = contLen; - - int tlen = tdEncodeKVRecord((void *)(&pBuf), &rInfo); - ASSERT(tlen == POINTER_DISTANCE(pBuf, buf)); - ASSERT(tlen == sizeof(SKVRecord)); - - if (taosWrite(pStore->fd, buf, tlen) < tlen) { - uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosWrite(pStore->fd, cont, contLen) < contLen) { - uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pStore->info.magic = - taosCalcChecksum(pStore->info.magic, (uint8_t *)POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - pStore->info.size += (sizeof(SKVRecord) + contLen); - SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); - if (pRecord != NULL) { // just to insert - pStore->info.tombSize += pRecord->size; - } else { - pStore->info.nRecords++; - } - - taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); - uTrace("put uid %" PRIu64 " into kvStore %s", uid, pStore->fname); - - return 0; -} - -int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { - SKVRecord rInfo = {0}; - char buf[128] = "\0"; - - SKVRecord *pRecord = taosHashGet(pStore->map, (void *)(&uid), sizeof(uid)); - if (pRecord == NULL) { - uError("failed to drop KV store record with key %" PRIu64 " since not find", uid); - return -1; - } - - rInfo.offset = -pRecord->offset; - rInfo.uid = pRecord->uid; - rInfo.size = pRecord->size; - - void *pBuf = buf; - tdEncodeKVRecord(&pBuf, &rInfo); - - if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { - uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pStore->info.magic = taosCalcChecksum(pStore->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf)); - pStore->info.size += POINTER_DISTANCE(pBuf, buf); - pStore->info.nDels++; - pStore->info.nRecords--; - pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - - taosHashRemove(pStore->map, (void *)(&uid), sizeof(uid)); - uDebug("drop uid %" PRIu64 " from KV store %s", uid, pStore->fname); - - return 0; -} - -int tdKVStoreEndCommit(SKVStore *pStore) { - ASSERT(pStore->fd > 0); - - if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &(pStore->info)) < 0) return -1; - - if (fsync(pStore->fd) < 0) { - uError("failed to fsync file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (close(pStore->fd) < 0) { - uError("failed to close file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pStore->fd = -1; - - (void)remove(pStore->fsnap); - return 0; -} - -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { - char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; - SStoreInfo info = {0}; - - int fd = open(fname, O_RDONLY); - if (fd < 0) goto _err; - - if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; - if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err; - - void *pBuf = (void *)buf; - pBuf = tdDecodeStoreInfo(pBuf, &info); - off_t offset = lseek(fd, 0, SEEK_END); - if (offset < 0) goto _err; - close(fd); - - *magic = info.magic; - *size = offset; - - return; - -_err: - if (fd >= 0) close(fd); - *magic = TD_KVSTORE_INIT_MAGIC; - *size = 0; -} - -static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version) { - char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; - - if (lseek(fd, 0, SEEK_SET) < 0) { - uError("failed to lseek file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { - uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) { - uError("file %s is broken", fname); - terrno = TSDB_CODE_COM_FILE_CORRUPTED; - return -1; - } - - void *pBuf = (void *)buf; - pBuf = tdDecodeStoreInfo(pBuf, pInfo); - pBuf = taosDecodeFixedU32(pBuf, version); - - return 0; -} - -static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { - char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; - - if (lseek(fd, 0, SEEK_SET) < 0) { - uError("failed to lseek file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - void *pBuf = buf; - tdEncodeStoreInfo(&pBuf, pInfo); - taosEncodeFixedU32(&pBuf, KVSTORE_FILE_VERSION); - ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE); - if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { - uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static int tdInitKVStoreHeader(int fd, char *fname) { - SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0, TD_KVSTORE_INIT_MAGIC}; - - return tdUpdateKVStoreHeader(fd, fname, &info); -} - -static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) { - int tlen = 0; - tlen += taosEncodeVariantI64(buf, pInfo->size); - tlen += taosEncodeVariantI64(buf, pInfo->tombSize); - tlen += taosEncodeVariantI64(buf, pInfo->nRecords); - tlen += taosEncodeVariantI64(buf, pInfo->nDels); - tlen += taosEncodeFixedU32(buf, pInfo->magic); - - return tlen; -} - -static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { - buf = taosDecodeVariantI64(buf, &(pInfo->size)); - buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); - buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); - buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); - buf = taosDecodeFixedU32(buf, &(pInfo->magic)); - - return buf; -} - -static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) { - SKVStore *pStore = (SKVStore *)calloc(1, sizeof(SKVStore)); - if (pStore == NULL) goto _err; - - pStore->fname = strdup(fname); - if (pStore->fname == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - pStore->fsnap = tdGetKVStoreSnapshotFname(fname); - if (pStore->fsnap == NULL) { - goto _err; - } - - pStore->fnew = tdGetKVStoreNewFname(fname); - if (pStore->fnew == NULL) goto _err; - - pStore->fd = -1; - pStore->sfd = -1; - pStore->nfd = -1; - pStore->iFunc = iFunc; - pStore->aFunc = aFunc; - pStore->appH = appH; - pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (pStore->map == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - return pStore; - -_err: - tdFreeKVStore(pStore); - return NULL; -} - -static void tdFreeKVStore(SKVStore *pStore) { - if (pStore) { - tfree(pStore->fname); - tfree(pStore->fsnap); - tfree(pStore->fnew); - taosHashCleanup(pStore->map); - free(pStore); - } -} - -static char *tdGetKVStoreSnapshotFname(char *fdata) { - size_t size = strlen(fdata) + strlen(TD_KVSTORE_SNAP_SUFFIX) + 1; - char * fname = malloc(size); - if (fname == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - return NULL; - } - sprintf(fname, "%s%s", fdata, TD_KVSTORE_SNAP_SUFFIX); - return fname; -} - -static char *tdGetKVStoreNewFname(char *fdata) { - size_t size = strlen(fdata) + strlen(TD_KVSTORE_NEW_SUFFIX) + 1; - char * fname = malloc(size); - if (fname == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - return NULL; - } - sprintf(fname, "%s%s", fdata, TD_KVSTORE_NEW_SUFFIX); - return fname; -} - -static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord) { - int tlen = 0; - tlen += taosEncodeFixedU64(buf, pRecord->uid); - tlen += taosEncodeFixedI64(buf, pRecord->offset); - tlen += taosEncodeFixedI64(buf, pRecord->size); - - return tlen; -} - -static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) { - buf = taosDecodeFixedU64(buf, &(pRecord->uid)); - buf = taosDecodeFixedI64(buf, &(pRecord->offset)); - buf = taosDecodeFixedI64(buf, &(pRecord->size)); - - return buf; -} - -static int tdRestoreKVStore(SKVStore *pStore) { - char tbuf[128] = "\0"; - void * buf = NULL; - int64_t maxBufSize = 0; - SKVRecord rInfo = {0}; - SKVRecord *pRecord = NULL; - - ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR)); - ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); - - while (true) { - int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord)); - if (tsize == 0) break; - if (tsize < sizeof(SKVRecord)) { - uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, - pStore->info.size, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - char *pBuf = tdDecodeKVRecord(tbuf, &rInfo); - ASSERT(POINTER_DISTANCE(pBuf, tbuf) == sizeof(SKVRecord)); - ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true); - - if (rInfo.offset < 0) { - taosHashRemove(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid)); - pStore->info.size += sizeof(SKVRecord); - pStore->info.nRecords--; - pStore->info.nDels++; - pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - } else { - ASSERT(rInfo.offset > 0 && rInfo.size > 0); - if (taosHashPut(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) { - uError("failed to put record in KV store %s", pStore->fname); - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - maxBufSize = MAX(maxBufSize, rInfo.size); - - if (lseek(pStore->fd, (off_t)rInfo.size, SEEK_CUR) < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pStore->info.size += (sizeof(SKVRecord) + rInfo.size); - pStore->info.nRecords++; - } - } - - buf = malloc((size_t)maxBufSize); - if (buf == NULL) { - uError("failed to allocate %" PRId64 " bytes in KV store %s", maxBufSize, pStore->fname); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pRecord = taosHashIterate(pStore->map, NULL); - while (pRecord) { - if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) { - uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) { - uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname, - strerror(errno), pRecord->offset); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (pStore->iFunc) { - if ((*pStore->iFunc)(pStore->appH, buf, (int)pRecord->size) < 0) { - uError("failed to restore record uid %" PRIu64 " in kv store %s at offset %" PRId64 " size %" PRId64 - " since %s", - pRecord->uid, pStore->fname, pRecord->offset, pRecord->size, tstrerror(terrno)); - goto _err; - } - } - - pRecord = taosHashIterate(pStore->map, pRecord); - } - - if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); - - tfree(buf); - return 0; - -_err: - taosHashCancelIterate(pStore->map, pRecord); - tfree(buf); - return -1; -} diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index 8c2ad83de117aaf528565a711a2aa3732984e0a9..2f52551e2ac7c79ef54cd4546be7bf844ff04980 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -38,11 +38,13 @@ void tdListEmpty(SList *list) { list->numOfEles = 0; } -void tdListFree(SList *list) { +void *tdListFree(SList *list) { if (list) { tdListEmpty(list); free(list); } + + return NULL; } void tdListPrependNode(SList *list, SListNode *node) { diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index ad3a92230482ff07b1533ceaf22b9ca30f72e10b..448bb3ecd1bfa8ef17bd7c87a464c095e1093d6d 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -17,6 +17,7 @@ #include "os.h" #include "tulog.h" #include "tlog.h" +#include "tnote.h" #include "tutil.h" #define MAX_LOGLINE_SIZE (1000) @@ -27,10 +28,13 @@ #define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) #define LOG_FILE_NAME_LEN 300 -#define TSDB_DEFAULT_LOG_BUF_SIZE (512 * 1024) // 512K -#define TSDB_MIN_LOG_BUF_SIZE 1024 // 1K -#define TSDB_MAX_LOG_BUF_SIZE (1024 * 1024) // 1M -#define TSDB_DEFAULT_LOG_BUF_UNIT 1024 // 1K +#define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB + +#define DEFAULT_LOG_INTERVAL 25 +#define LOG_INTERVAL_STEP 5 +#define MIN_LOG_INTERVAL 5 +#define MAX_LOG_INTERVAL 25 +#define LOG_MAX_WAIT_MSEC 1000 #define LOG_BUF_BUFFER(x) ((x)->buffer) #define LOG_BUF_START(x) ((x)->buffStart) @@ -43,6 +47,7 @@ typedef struct { int32_t buffStart; int32_t buffEnd; int32_t buffSize; + int32_t minBuffSize; int32_t fd; int32_t stop; pthread_t asyncThread; @@ -63,10 +68,19 @@ typedef struct { } SLogObj; int32_t tsLogKeepDays = 0; -int32_t tsAsyncLog = 1; +int8_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 0.1f; +float tsMinimalLogDirGB = 1.0f; +int64_t asyncLogLostLines = 0; +int32_t writeInterval = DEFAULT_LOG_INTERVAL; + +int64_t dbgEmptyW = 0; +int64_t dbgWN = 0; +int64_t dbgSmallWN = 0; +int64_t dbgBigWN = 0; +int64_t dbgWSize = 0; + #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; #else @@ -107,7 +121,8 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); - tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); + //tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); + taosMsleep(MAX_LOG_INTERVAL/1000); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { pthread_join(tsLogObj.logHandle->asyncThread, NULL); } @@ -170,7 +185,9 @@ static void *taosThreadToOpenNewFile(void *param) { int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { - uError("open new log file fail! fd:%d reason:%s", fd, strerror(errno)); + tsLogObj.openInProgress = 0; + tsLogObj.lines = tsLogObj.maxLines - 1000; + uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno)); return NULL; } @@ -287,7 +304,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { tsLogObj.fileNum = maxFileNum; taosGetLogFileName(fn); - if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { strcpy(name, fn); strcat(name, ".0"); @@ -363,8 +379,8 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); @@ -401,6 +417,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len); + if (dflag == 255) nInfo(buffer, len); } void taosDumpData(unsigned char *msg, int32_t len) { @@ -448,8 +465,8 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); @@ -494,8 +511,6 @@ static void taosCloseLogByFd(int32_t fd) { static SLogBuff *taosLogBuffNew(int32_t bufSize) { SLogBuff *tLogBuff = NULL; - if (bufSize < TSDB_MIN_LOG_BUF_SIZE || bufSize > TSDB_MAX_LOG_BUF_SIZE) return NULL; - tLogBuff = calloc(1, sizeof(SLogBuff)); if (tLogBuff == NULL) return NULL; @@ -504,10 +519,11 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { LOG_BUF_START(tLogBuff) = LOG_BUF_END(tLogBuff) = 0; LOG_BUF_SIZE(tLogBuff) = bufSize; + tLogBuff->minBuffSize = bufSize / 10; tLogBuff->stop = 0; if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; - tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); + //tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; @@ -526,10 +542,27 @@ static void taosLogBuffDestroy(SLogBuff *tLogBuff) { } #endif +static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, char *msg, int32_t msgLen) { + if (start > end) { + memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); + } else { + if (LOG_BUF_SIZE(tLogBuff) - end < msgLen) { + memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, LOG_BUF_SIZE(tLogBuff) - end); + memcpy(LOG_BUF_BUFFER(tLogBuff), msg + LOG_BUF_SIZE(tLogBuff) - end, msgLen - LOG_BUF_SIZE(tLogBuff) + end); + } else { + memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); + } + } + LOG_BUF_END(tLogBuff) = (LOG_BUF_END(tLogBuff) + msgLen) % LOG_BUF_SIZE(tLogBuff); +} + static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) { int32_t start = 0; int32_t end = 0; int32_t remainSize = 0; + static int64_t lostLine = 0; + char tmpBuf[40] = {0}; + int32_t tmpBufLen = 0; if (tLogBuff == NULL || tLogBuff->stop) return -1; @@ -537,79 +570,128 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) start = LOG_BUF_START(tLogBuff); end = LOG_BUF_END(tLogBuff); - remainSize = (start > end) ? (end - start - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); + remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); + + if (lostLine > 0) { + sprintf(tmpBuf, "...Lost %"PRId64" lines here...\n", lostLine); + tmpBufLen = (int32_t)strlen(tmpBuf); + } - if (remainSize <= msgLen) { + if (remainSize <= msgLen || ((lostLine > 0) && (remainSize <= (msgLen + tmpBufLen)))) { + lostLine++; + asyncLogLostLines++; pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); return -1; } - if (start > end) { - memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); - } else { - if (LOG_BUF_SIZE(tLogBuff) - end < msgLen) { - memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, LOG_BUF_SIZE(tLogBuff) - end); - memcpy(LOG_BUF_BUFFER(tLogBuff), msg + LOG_BUF_SIZE(tLogBuff) - end, msgLen - LOG_BUF_SIZE(tLogBuff) + end); - } else { - memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); - } + if (lostLine > 0) { + taosCopyLogBuffer(tLogBuff, start, end, tmpBuf, tmpBufLen); + lostLine = 0; } - LOG_BUF_END(tLogBuff) = (LOG_BUF_END(tLogBuff) + msgLen) % LOG_BUF_SIZE(tLogBuff); - // TODO : put string in the buffer + taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen); - tsem_post(&(tLogBuff->buffNotEmpty)); + //int32_t w = atomic_sub_fetch_32(&waitLock, 1); + /* + if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) { + tsem_post(&(tLogBuff->buffNotEmpty)); + dbgPostN++; + } else { + dbgNoPostN++; + } + */ pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + return 0; } -static int32_t taosPollLogBuffer(SLogBuff *tLogBuff, char *buf, int32_t bufSize) { - int32_t start = LOG_BUF_START(tLogBuff); - int32_t end = LOG_BUF_END(tLogBuff); - int32_t pollSize = 0; +static int32_t taosGetLogRemainSize(SLogBuff *tLogBuff, int32_t start, int32_t end) { + int32_t rSize = end - start; - if (start == end) { - return 0; - } else if (start < end) { - pollSize = MIN(end - start, bufSize); + return rSize >= 0 ? rSize : LOG_BUF_SIZE(tLogBuff) + rSize; +} - memcpy(buf, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); - return pollSize; - } else { - pollSize = MIN(end + LOG_BUF_SIZE(tLogBuff) - start, bufSize); - if (pollSize > LOG_BUF_SIZE(tLogBuff) - start) { - int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - memcpy(buf, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - memcpy(buf + tsize, LOG_BUF_BUFFER(tLogBuff), pollSize - tsize); +static void taosWriteLog(SLogBuff *tLogBuff) { + static int32_t lastDuration = 0; + int32_t remainChecked = 0; + int32_t start, end, pollSize; + + do { + if (remainChecked == 0) { + start = LOG_BUF_START(tLogBuff); + end = LOG_BUF_END(tLogBuff); + + if (start == end) { + dbgEmptyW++; + writeInterval = MAX_LOG_INTERVAL; + return; + } + pollSize = taosGetLogRemainSize(tLogBuff, start, end); + if (pollSize < tLogBuff->minBuffSize) { + lastDuration += writeInterval; + if (lastDuration < LOG_MAX_WAIT_MSEC) { + break; + } + } + + lastDuration = 0; + } + + if (start < end) { + taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { - memcpy(buf, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; + taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + + taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); } - return pollSize; - } + + dbgWN++; + dbgWSize+=pollSize; + + if (pollSize < tLogBuff->minBuffSize) { + dbgSmallWN++; + if (writeInterval < MAX_LOG_INTERVAL) { + writeInterval += LOG_INTERVAL_STEP; + } + } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/3) { + dbgBigWN++; + writeInterval = MIN_LOG_INTERVAL; + } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/4) { + if (writeInterval > MIN_LOG_INTERVAL) { + writeInterval -= LOG_INTERVAL_STEP; + } + } + + LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + pollSize) % LOG_BUF_SIZE(tLogBuff); + + start = LOG_BUF_START(tLogBuff); + end = LOG_BUF_END(tLogBuff); + + pollSize = taosGetLogRemainSize(tLogBuff, start, end); + if (pollSize < tLogBuff->minBuffSize) { + break; + } + + writeInterval = MIN_LOG_INTERVAL; + + remainChecked = 1; + }while (1); } static void *taosAsyncOutputLog(void *param) { SLogBuff *tLogBuff = (SLogBuff *)param; - int32_t log_size = 0; - - char tempBuffer[TSDB_DEFAULT_LOG_BUF_UNIT]; - + while (1) { - tsem_wait(&(tLogBuff->buffNotEmpty)); + //tsem_wait(&(tLogBuff->buffNotEmpty)); + + taosMsleep(writeInterval); // Polling the buffer - while (1) { - log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT); - if (log_size) { - taosWrite(tLogBuff->fd, tempBuffer, log_size); - LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff); - } else { - break; - } - } + taosWriteLog(tLogBuff); if (tLogBuff->stop) break; } diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 6fd526598365f831addd1bacb0b7f748d9552fdb..28abad356c156182b23feebd8d2e80ea4c914e52 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -13,50 +13,45 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" #include "tulog.h" -#include "tconfig.h" #include "tglobal.h" #include "tsocket.h" #include "trpc.h" #include "rpcHead.h" -#include "tutil.h" -#include "tnettest.h" +#include "tchecksum.h" +#include "syncMsg.h" -#define MAX_PKG_LEN (64*1000) -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) +#define MAX_PKG_LEN (64 * 1000) +#define BUFFER_SIZE (MAX_PKG_LEN + 1024) + +extern int32_t tsRpcMaxUdpSize; typedef struct { + char * hostFqdn; uint32_t hostIp; - uint16_t port; - uint16_t pktLen; -} info_s; - -extern int tsRpcMaxUdpSize; - -static char g_user[TSDB_USER_LEN+1] = {0}; -static char g_pass[TSDB_PASSWORD_LEN+1] = {0}; -static char g_serverFqdn[TSDB_FQDN_LEN] = {0}; -static uint16_t g_startPort = 0; -static uint16_t g_endPort = 6042; -static uint32_t g_pktLen = 0; - - -static void *bindUdpPort(void *sarg) { - info_s *pinfo = (info_s *)sarg; - int port = pinfo->port; - SOCKET serverSocket; + int32_t port; + int32_t pktLen; +} STestInfo; + +static void *taosNetBindUdpPort(void *sarg) { + STestInfo *pinfo = (STestInfo *)sarg; + int32_t port = pinfo->port; + SOCKET serverSocket; + char buffer[BUFFER_SIZE]; + int32_t iDataNum; + socklen_t sin_size; + int32_t bufSize = 1024000; struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - char buffer[BUFFER_SIZE]; - int iDataNum; - - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - perror("socket"); + + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -66,49 +61,60 @@ static void *bindUdpPort(void *sarg) { server_addr.sin_addr.s_addr = htonl(INADDR_ANY); if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - perror("connect"); + uError("failed to bind UDP port:%d since %s", port, strerror(errno)); return NULL; } - socklen_t sin_size; + if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(serverSocket); + return NULL; + } + + if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(serverSocket); + return NULL; + } + + uInfo("UDP server at port:%d is listening", port); while (1) { memset(buffer, 0, BUFFER_SIZE); - sin_size = sizeof(*(struct sockaddr *)&server_addr); - iDataNum = recvfrom(serverSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&clientAddr, &sin_size); if (iDataNum < 0) { - perror("recvfrom null"); + uDebug("failed to perform recvfrom func at %d since %s", port, strerror(errno)); continue; } - if (iDataNum > 0) { - printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum); - //printf("Read msg from udp:%s ... %s\n", buffer, buffer+iDataNum-16); - sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int)sin_size); + uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); + + if (iDataNum > 0) { + iDataNum = taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); } + + uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); } taosCloseSocket(serverSocket); return NULL; } -static void *bindTcpPort(void *sarg) { - info_s *pinfo = (info_s *)sarg; - int port = pinfo->port; - SOCKET serverSocket; - +static void *taosNetBindTcpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - int addr_len = sizeof(clientAddr); - SOCKET client; - char buffer[BUFFER_SIZE]; - int iDataNum = 0; + + STestInfo *pinfo = sarg; + int32_t port = pinfo->port; + SOCKET serverSocket; + int32_t addr_len = sizeof(clientAddr); + SOCKET client; + char buffer[BUFFER_SIZE]; if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { - printf("socket() fail: %s", strerror(errno)); + uError("failed to create TCP socket since %s", strerror(errno)); return NULL; } @@ -117,235 +123,204 @@ static void *bindTcpPort(void *sarg) { server_addr.sin_port = htons(port); server_addr.sin_addr.s_addr = htonl(INADDR_ANY); + int32_t reuse = 1; + if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(serverSocket); + return NULL; + } + if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - printf("port:%d bind() fail: %s", port, strerror(errno)); + uError("failed to bind TCP port:%d since %s", port, strerror(errno)); return NULL; } - if (listen(serverSocket, 5) < 0) { - printf("listen() fail: %s", strerror(errno)); + if (taosKeepTcpAlive(serverSocket) < 0) { + uError("failed to set tcp server keep-alive option since %s", strerror(errno)); + taosCloseSocket(serverSocket); return NULL; } - //printf("Bind port: %d success\n", port); + if (listen(serverSocket, 10) < 0) { + uError("failed to listen TCP port:%d since %s", port, strerror(errno)); + return NULL; + } + + uInfo("TCP server at port:%d is listening", port); + while (1) { client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); if (client < 0) { - printf("accept() fail: %s", strerror(errno)); + uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); continue; } - iDataNum = 0; - memset(buffer, 0, BUFFER_SIZE); - int nleft, nread; - char *ptr = buffer; - nleft = pinfo->pktLen; - while (nleft > 0) { - nread = recv(client, ptr, BUFFER_SIZE, 0); - - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR) { - continue; - } else { - printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", taosInetNtoa(clientAddr.sin_addr), port, strerror(errno)); - taosCloseSocket(serverSocket); - return NULL; - } - } else { - nleft -= nread; - ptr += nread; - iDataNum += nread; - } + int32_t ret = taosReadMsg(client, buffer, pinfo->pktLen); + if (ret < 0 || ret != pinfo->pktLen) { + uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno)); + taosCloseSocket(serverSocket); + return NULL; } - - printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum); - if (iDataNum > 0) { - send(client, buffer, iDataNum, 0); + + uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); + + ret = taosWriteMsg(client, buffer, pinfo->pktLen); + if (ret < 0) { + uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno)); + taosCloseSocket(serverSocket); + return NULL; } + + uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); } - + taosCloseSocket(serverSocket); return NULL; } -static int checkTcpPort(info_s *info) { - struct sockaddr_in serverAddr; - SOCKET clientSocket; - char sendbuf[BUFFER_SIZE]; - char recvbuf[BUFFER_SIZE]; - int iDataNum = 0; +static int32_t taosNetCheckTcpPort(STestInfo *info) { + SOCKET clientSocket; + char buffer[BUFFER_SIZE] = {0}; + if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - printf("socket() fail: %s\n", strerror(errno)); + uError("failed to create TCP client socket since %s", strerror(errno)); return -1; } - // set send and recv overtime - struct timeval timeout; - timeout.tv_sec = 2; //s - timeout.tv_usec = 0; //us - if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt send timer failed:"); - } - if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt recv timer failed:"); + int32_t reuse = 1; + if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(clientSocket); + return -1; } + struct sockaddr_in serverAddr; + memset((char *)&serverAddr, 0, sizeof(serverAddr)); serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(info->port); - + serverAddr.sin_port = (uint16_t)htons((uint16_t)info->port); serverAddr.sin_addr.s_addr = info->hostIp; - //printf("=================================\n"); if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - printf("connect() fail: %s\t", strerror(errno)); + uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); return -1; } - //printf("Connect to: %s:%d...success\n", host, port); - memset(sendbuf, 0, BUFFER_SIZE); - memset(recvbuf, 0, BUFFER_SIZE); - struct in_addr ipStr; - memcpy(&ipStr, &info->hostIp, 4); - sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); - sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); - - send(clientSocket, sendbuf, info->pktLen, 0); - - memset(recvbuf, 0, BUFFER_SIZE); - int nleft, nread; - char *ptr = recvbuf; - nleft = info->pktLen; - while (nleft > 0) { - nread = recv(clientSocket, ptr, BUFFER_SIZE, 0);; - - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR) { - continue; - } else { - printf("recv ack pkg from TCP port: %d fail:%s.\n", info->port, strerror(errno)); - taosCloseSocket(clientSocket); - return -1; - } - } else { - nleft -= nread; - ptr += nread; - iDataNum += nread; - } + taosKeepTcpAlive(clientSocket); + + sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port); + sprintf(buffer + info->pktLen - 16, "1122334455667788"); + + int32_t ret = taosWriteMsg(clientSocket, buffer, info->pktLen); + if (ret < 0) { + uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + return -1; } - if (iDataNum < info->pktLen) { - printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, info->port); + ret = taosReadMsg(clientSocket, buffer, info->pktLen); + if (ret < 0) { + uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); return -1; } - //printf("Read ack pkg len:%d from tcp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); taosCloseSocket(clientSocket); return 0; } -static int checkUdpPort(info_s *info) { +static int32_t taosNetCheckUdpPort(STestInfo *info) { + SOCKET clientSocket; + char buffer[BUFFER_SIZE] = {0}; + int32_t iDataNum = 0; + int32_t bufSize = 1024000; + struct sockaddr_in serverAddr; - SOCKET clientSocket; - char sendbuf[BUFFER_SIZE]; - char recvbuf[BUFFER_SIZE]; - int iDataNum = 0; + if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - perror("socket"); + uError("failed to create udp client socket since %s", strerror(errno)); return -1; } - // set overtime - struct timeval timeout; - timeout.tv_sec = 2; //s - timeout.tv_usec = 0; //us - if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt send timer failed:"); + if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the send buffer size for UDP socket\n"); + return -1; } - if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt recv timer failed:"); + + if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the receive buffer size for UDP socket\n"); + return -1; } - + serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(info->port); serverAddr.sin_addr.s_addr = info->hostIp; - - memset(sendbuf, 0, BUFFER_SIZE); - memset(recvbuf, 0, BUFFER_SIZE); struct in_addr ipStr; memcpy(&ipStr, &info->hostIp, 4); - sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); - sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); + sprintf(buffer, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); + sprintf(buffer + info->pktLen - 16, "1122334455667788"); socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - int code = sendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int)sin_size); - if (code < 0) { - perror("sendto"); + iDataNum = taosSendto(clientSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); + if (iDataNum < 0 || iDataNum != info->pktLen) { + uError("UDP: failed to perform sendto func since %s", strerror(errno)); return -1; } - iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); + memset(buffer, 0, BUFFER_SIZE); + sin_size = sizeof(*(struct sockaddr *)&serverAddr); + iDataNum = recvfrom(clientSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); - if (iDataNum < info->pktLen) { - printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\t\t", iDataNum, info->pktLen, info->port); + if (iDataNum < 0 || iDataNum != info->pktLen) { + uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno)); return -1; } - - //printf("Read ack pkg len:%d from udp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); + taosCloseSocket(clientSocket); return 0; } -static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t pktLen) { - int ret; - info_s info; - memset(&info, 0, sizeof(info_s)); - info.hostIp = hostIp; - info.pktLen = pktLen; +static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort, int32_t pktLen) { + int32_t ret; + STestInfo info; - for (uint16_t port = startPort; port <= maxPort; port++) { - //printf("test: %s:%d\n", info.host, port); - printf("\n"); + memset(&info, 0, sizeof(STestInfo)); + info.hostIp = hostIp; + info.pktLen = pktLen; + for (int32_t port = startPort; port <= endPort; port++) { info.port = port; - ret = checkTcpPort(&info); + ret = taosNetCheckTcpPort(&info); if (ret != 0) { - printf("tcp port:%d test fail.\t\n", port); + uError("failed to test TCP port:%d", port); } else { - printf("tcp port:%d test ok.\t\t", port); + uInfo("successed to test TCP port:%d", port); } - - ret = checkUdpPort(&info); + + ret = taosNetCheckUdpPort(&info); if (ret != 0) { - printf("udp port:%d test fail.\t\n", port); + uError("failed to test UDP port:%d", port); } else { - printf("udp port:%d test ok.\t\t", port); + uInfo("successed to test UDP port:%d", port); } } - - printf("\n"); - return ; } -void* tnetInitRpc(char* secretEncrypt, char spi) { +void *taosNetInitRpc(char *secretEncrypt, char spi) { SRpcInit rpcInit; - void* pRpcConn = NULL; + void * pRpcConn = NULL; + + char user[] = "nettestinternal"; + char pass[] = "nettestinternal"; + taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); - taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt); - memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; - rpcInit.label = "NET-TEST"; + rpcInit.label = "NT"; rpcInit.numOfThreads = 1; // every DB connection has only one thread rpcInit.cfp = NULL; rpcInit.sessions = 16; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = g_user; + rpcInit.user = user; rpcInit.idleTime = 2000; rpcInit.ckey = "key"; rpcInit.spi = spi; @@ -355,17 +330,18 @@ void* tnetInitRpc(char* secretEncrypt, char spi) { return pRpcConn; } -static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) { +static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupStep *pStep) { SRpcEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; - void* pRpcConn; + void * pRpcConn; char secretEncrypt[32] = {0}; - pRpcConn = tnetInitRpc(secretEncrypt, spi); + pRpcConn = taosNetInitRpc(secretEncrypt, spi); if (NULL == pRpcConn) { - return -1; + uError("failed to init client rpc"); + return TSDB_CODE_RPC_NETWORK_UNAVAIL; } memset(&epSet, 0, sizeof(SRpcEpSet)); @@ -373,205 +349,215 @@ static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktL epSet.numOfEps = 1; epSet.port[0] = port; strcpy(epSet.fqdn[0], serverFqdn); - + reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pktLen); reqMsg.contLen = pktLen; reqMsg.code = 0; reqMsg.handle = NULL; // rpc handle returned to app - reqMsg.ahandle = NULL; // app handle set by client + reqMsg.ahandle = NULL; // app handle set by client + strcpy(reqMsg.pCont, "nettest"); rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); - // handle response if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) { - //printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code)); - return -1; + uDebug("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code)); + return rspMsg.code; + } + + int32_t code = 0; + if (pStep != NULL && rspMsg.pCont != NULL && rspMsg.contLen > 0 && rspMsg.contLen <= sizeof(SStartupStep)) { + memcpy(pStep, rspMsg.pCont, rspMsg.contLen); + code = 1; } - - rpcFreeCont(rspMsg.pCont); + rpcFreeCont(rspMsg.pCont); rpcClose(pRpcConn); + return code; +} - return 0; +static int32_t taosNetParseStartup(SStartupStep *pCont) { + SStartupStep *pStep = pCont; + uInfo("step:%s desc:%s", pStep->name, pStep->desc); + + if (pStep->finished) { + uInfo("check startup finished"); + } + + return pStep->finished ? 0 : 1; } -static void rpcCheckPort(uint32_t hostIp) { - int ret; - char spi; +static void taosNetTestStartup(char *host, int32_t port) { + uInfo("check startup, host:%s port:%d\n", host, port); - for (uint16_t port = g_startPort; port <= g_endPort; port++) { - //printf("test: %s:%d\n", info.host, port); - printf("\n"); + SStartupStep *pStep = malloc(sizeof(SStartupStep)); + while (1) { + int32_t code = taosNetCheckRpc(host, port + TSDB_PORT_DNODEDNODE, 20, 0, pStep); + if (code > 0) { + code = taosNetParseStartup(pStep); + } - //================ check tcp port ================ - int32_t pktLen; - if (g_pktLen <= tsRpcMaxUdpSize) { - pktLen = tsRpcMaxUdpSize + 1000; + if (code > 0) { + uDebug("continue check startup step"); } else { - pktLen = g_pktLen; + if (code < 0) { + uError("failed to check startup step, code:0x%x %s", code, tstrerror(code)); + } + break; } + } - spi = 1; - ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); - if (ret != 0) { - spi = 0; - ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); - if (ret != 0) { - printf("TCP port:%d test fail.\t\t", port); - } else { - //printf("tcp port:%d test ok.\t\t", port); - printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); - } + free(pStep); +} + +static void taosNetCheckSync(char *host, int32_t port) { + uint32_t ip = taosGetIpv4FromFqdn(host); + if (ip == 0xffffffff) { + uError("failed to get IP address from %s since %s", host, strerror(errno)); + return; + } + + SOCKET connFd = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); + if (connFd < 0) { + uError("failed to create socket while test port:%d since %s", port, strerror(errno)); + return; + } + + SSyncMsg msg; + memset(&msg, 0, sizeof(SSyncMsg)); + SSyncHead *pHead = &msg.head; + pHead->type = TAOS_SMSG_TEST; + pHead->protocol = SYNC_PROTOCOL_VERSION; + pHead->signature = SYNC_SIGNATURE; + pHead->code = 0; + pHead->cId = 0; + pHead->vgId = -1; + pHead->len = sizeof(SSyncMsg) - sizeof(SSyncHead); + taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); + + if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + uError("failed to test port:%d while send msg since %s", port, strerror(errno)); + return; + } + + if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + uError("failed to test port:%d while recv msg since %s", port, strerror(errno)); + } + + uInfo("successed to test TCP port:%d", port); + taosCloseSocket(connFd); +} + +static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { + int32_t endPort = startPort + TSDB_PORT_SYNC; + char spi = 0; + + uInfo("check rpc, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + + for (uint16_t port = startPort; port < endPort; port++) { + int32_t sendpkgLen; + if (pkgLen <= tsRpcMaxUdpSize) { + sendpkgLen = tsRpcMaxUdpSize + 1000; } else { - //printf("tcp port:%d test ok.\t\t", port); - printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); + sendpkgLen = pkgLen; } - //================ check udp port ================ - if (g_pktLen >= tsRpcMaxUdpSize) { - pktLen = tsRpcMaxUdpSize - 1000; + int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); + if (ret < 0) { + uError("failed to test TCP port:%d", port); } else { - pktLen = g_pktLen; + uInfo("successed to test TCP port:%d", port); } - - spi = 0; - ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); - if (ret != 0) { - spi = 1; - ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); - if (ret != 0) { - printf("udp port:%d test fail.\t\n", port); - } else { - //printf("udp port:%d test ok.\t\n", port); - printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); - } + + if (pkgLen >= tsRpcMaxUdpSize) { + sendpkgLen = tsRpcMaxUdpSize - 1000; + } else { + sendpkgLen = pkgLen; + } + + ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); + if (ret < 0) { + uError("failed to test UDP port:%d", port); } else { - //printf("udp port:%d test ok.\t\n", port); - printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); + uInfo("successed to test UDP port:%d", port); } } - - printf("\n"); - return ; + + taosNetCheckSync(host, startPort + TSDB_PORT_SYNC); + taosNetCheckSync(host, startPort + TSDB_PORT_ARBITRATOR); } -static void taosNetTestClient(int flag) { - uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn); +static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { + int32_t endPort = startPort + 11; + uInfo("work as client, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + + uint32_t serverIp = taosGetIpv4FromFqdn(host); if (serverIp == 0xFFFFFFFF) { - printf("Failed to resolve FQDN:%s", g_serverFqdn); + uError("failed to resolve fqdn:%s", host); exit(-1); } - if (0 == flag) { - checkPort(serverIp, g_startPort, g_endPort, g_pktLen); - } else { - rpcCheckPort(serverIp); - } - - return; + uInfo("server ip:%s is resolved from host:%s", taosIpStr(serverIp), host); + taosNetCheckPort(serverIp, startPort, endPort, pkgLen); } -static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) { +static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { + int32_t endPort = startPort + 11; + uInfo("work as server, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); - int port = startPort; - int num = endPort - startPort + 1; + int32_t port = startPort; + int32_t num = endPort - startPort + 1; + if (num < 0) num = 1; - if (num < 0) { - num = 1; - } - pthread_t *pids = malloc(2 * num * sizeof(pthread_t)); - info_s * tinfos = malloc(num * sizeof(info_s)); - info_s * uinfos = malloc(num * sizeof(info_s)); + STestInfo *tinfos = malloc(num * sizeof(STestInfo)); + STestInfo *uinfos = malloc(num * sizeof(STestInfo)); - for (size_t i = 0; i < num; i++) { - info_s *tcpInfo = tinfos + i; - tcpInfo->port = (uint16_t)(port + i); - tcpInfo->pktLen = pktLen; + for (int32_t i = 0; i < num; i++) { + STestInfo *tcpInfo = tinfos + i; + tcpInfo->port = port + i; + tcpInfo->pktLen = pkgLen; - if (pthread_create(pids + i, NULL, bindTcpPort, tcpInfo) != 0) - { - printf("create thread fail, port:%d.\n", port); + if (pthread_create(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { + uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } - info_s *udpInfo = uinfos + i; - udpInfo->port = (uint16_t)(port + i); - if (pthread_create(pids + num + i, NULL, bindUdpPort, udpInfo) != 0) - { - printf("create thread fail, port:%d.\n", port); + STestInfo *udpInfo = uinfos + i; + udpInfo->port = port + i; + tcpInfo->pktLen = pkgLen; + if (pthread_create(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { + uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } } - - for (int i = 0; i < num; i++) { + + for (int32_t i = 0; i < num; i++) { pthread_join(pids[i], NULL); pthread_join(pids[(num + i)], NULL); } } - -void taosNetTest(CmdArguments *args) { - if (0 == args->pktLen) { - g_pktLen = 1000; +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) { + tscEmbedded = 1; + if (host == NULL) host = tsLocalFqdn; + if (port == 0) port = tsServerPort; + if (pkgLen <= 10) pkgLen = 1000; + if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN; + + if (0 == strcmp("client", role)) { + taosNetTestClient(host, port, pkgLen); + } else if (0 == strcmp("server", role)) { + taosNetTestServer(host, port, pkgLen); + } else if (0 == strcmp("rpc", role)) { + taosNetTestRpc(host, port, pkgLen); + } else if (0 == strcmp("sync", role)) { + taosNetCheckSync(host, port); + } else if (0 == strcmp("startup", role)) { + taosNetTestStartup(host, port); } else { - g_pktLen = args->pktLen; + taosNetTestStartup(host, port); } - - if (args->port && args->endPort) { - if (args->port > args->endPort) { - printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port); - exit(-1); - } - } - - if (args->host && args->host[0] != 0) { - if (strlen(args->host) >= TSDB_EP_LEN) { - printf("host invalid: %s\n", args->host); - exit(-1); - } - - taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort); - } else { - tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); - g_startPort = tsServerPort; - } - - if (args->port) { - g_startPort = args->port; - } - - if (args->endPort) { - g_endPort = args->endPort; - } - - if (g_startPort > g_endPort) { - printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort); - exit(-1); - } - - - if (args->is_use_passwd) { - if (args->password == NULL) args->password = getpass("Enter password: "); - } else { - args->password = TSDB_DEFAULT_PASS; - } - tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN); - if (args->user == NULL) { - args->user = TSDB_DEFAULT_USER; - } - tstrncpy(g_user, args->user, TSDB_USER_LEN); - - if (0 == strcmp("client", args->netTestRole)) { - printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); - taosNetTestClient(0); - } else if (0 == strcmp("clients", args->netTestRole)) { - printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); - taosNetTestClient(1); - } else if (0 == strcmp("server", args->netTestRole)) { - taosNetTestServer(g_startPort, g_endPort, g_pktLen); - } + tscEmbedded = 0; } - diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 9536f6fb70f9fe6d74c981274c849cab0c00ef1f..c6d49dfb3d8242bdf697b3212fde479f01f8bcd9 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -13,277 +13,260 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "tutil.h" +#include "tglobal.h" #include "tnote.h" -taosNoteInfo m_HttpNote; -taosNoteInfo m_TscNote; +SNoteObj tsHttpNote; +SNoteObj tsTscNote; +SNoteObj tsInfoNote; -int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote); +static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote); +static void taosCloseNoteByFd(int32_t oldFd, SNoteObj *pNote); -void taosInitNote(int numOfNoteLines, int maxNotes, char* lable) -{ - taosNoteInfo * pNote = NULL; - char temp[128] = { 0 }; +static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) { + memset(pNote, 0, sizeof(SNoteObj)); + pNote->fileNum = 1; + pNote->fd = -1; - if (strcasecmp(lable, "http_note") == 0) { - pNote = &m_HttpNote; - sprintf(temp, "%s/httpnote", tsLogDir); - } else if (strcasecmp(lable, "tsc_note") == 0) { - pNote = &m_TscNote; - sprintf(temp, "%s/tscnote-%d", tsLogDir, getpid()); - } else { - return; - } + if (taosOpenNoteWithMaxLines(name, numOfLines, maxNotes, pNote) < 0) { + fprintf(stderr, "failed to init note file\n"); + } + + taosNotePrint(pNote, "=================================================="); + taosNotePrint(pNote, "=================== new note ==================="); + taosNotePrint(pNote, "=================================================="); +} - memset(pNote, 0, sizeof(taosNoteInfo)); - pNote->taosNoteFileNum = 1; - //pNote->taosNoteMaxLines = 0; - //pNote->taosNoteLines = 0; - //pNote->taosNoteFlag = 0; - pNote->taosNoteFd = -1; - //pNote->taosNoteOpenInProgress = 0; +void taosInitNotes() { + char name[TSDB_FILENAME_LEN * 2] = {0}; - if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes, pNote) < 0) - fprintf(stderr, "failed to init note file\n"); + if (tsTscEnableRecordSql) { + snprintf(name, TSDB_FILENAME_LEN * 2, "%s/tscsql-%d", tsLogDir, taosGetPId()); + taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); + } - taosNotePrint(pNote, "=================================================="); - taosNotePrint(pNote, "=================== new note ==================="); - taosNotePrint(pNote, "=================================================="); + if (tsHttpEnableRecordSql) { + snprintf(name, TSDB_FILENAME_LEN * 2, "%s/httpsql", tsLogDir); + taosInitNote(tsNumOfLogLines, 1, &tsHttpNote, name); + } + + if (tscEmbedded == 1) { + snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir); + taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name); + } } -void taosCloseNoteByFd(int oldFd, taosNoteInfo * pNote); -bool taosLockNote(int fd, taosNoteInfo * pNote) -{ - if (fd < 0) return false; +static bool taosLockNote(int32_t fd, SNoteObj *pNote) { + if (fd < 0) return false; - if (pNote->taosNoteFileNum > 1) { - int ret = (int)(flock(fd, LOCK_EX | LOCK_NB)); - if (ret == 0) { - return true; - } + if (pNote->fileNum > 1) { + int32_t ret = (int32_t)(flock(fd, LOCK_EX | LOCK_NB)); + if (ret == 0) { + return true; } + } - return false; + return false; } -void taosUnLockNote(int fd, taosNoteInfo * pNote) -{ - if (fd < 0) return; +static void taosUnLockNote(int32_t fd, SNoteObj *pNote) { + if (fd < 0) return; - if (pNote->taosNoteFileNum > 1) { - flock(fd, LOCK_UN | LOCK_NB); - } + if (pNote->fileNum > 1) { + flock(fd, LOCK_UN | LOCK_NB); + } } -void *taosThreadToOpenNewNote(void *param) -{ - char name[NOTE_FILE_NAME_LEN * 2]; - taosNoteInfo * pNote = (taosNoteInfo *)param; +static void *taosThreadToOpenNewNote(void *param) { + char name[NOTE_FILE_NAME_LEN * 2]; + SNoteObj *pNote = (SNoteObj *)param; - pNote->taosNoteFlag ^= 1; - pNote->taosNoteLines = 0; - sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag); + pNote->flag ^= 1; + pNote->lines = 0; + sprintf(name, "%s.%d", pNote->name, pNote->flag); - umask(0); + umask(0); - int fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); - if (fd < 0) { - return NULL; - } + int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd < 0) { + return NULL; + } - taosLockNote(fd, pNote); - (void)lseek(fd, 0, SEEK_SET); + taosLockNote(fd, pNote); + (void)lseek(fd, 0, SEEK_SET); - int oldFd = pNote->taosNoteFd; - pNote->taosNoteFd = fd; - pNote->taosNoteLines = 0; - pNote->taosNoteOpenInProgress = 0; - taosNotePrint(pNote, "=============== new note is opened ============="); + int32_t oldFd = pNote->fd; + pNote->fd = fd; + pNote->lines = 0; + pNote->openInProgress = 0; + taosNotePrint(pNote, "=============== new note is opened ============="); - taosCloseNoteByFd(oldFd, pNote); - return NULL; + taosCloseNoteByFd(oldFd, pNote); + return NULL; } -int taosOpenNewNote(taosNoteInfo * pNote) -{ - pthread_mutex_lock(&pNote->taosNoteMutex); +static int32_t taosOpenNewNote(SNoteObj *pNote) { + pthread_mutex_lock(&pNote->mutex); - if (pNote->taosNoteLines > pNote->taosNoteMaxLines && pNote->taosNoteOpenInProgress == 0) { - pNote->taosNoteOpenInProgress = 1; + if (pNote->lines > pNote->maxLines && pNote->openInProgress == 0) { + pNote->openInProgress = 1; - taosNotePrint(pNote, "=============== open new note =================="); - pthread_t pattern; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + taosNotePrint(pNote, "=============== open new note =================="); + pthread_t pattern; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void*)pNote); - pthread_attr_destroy(&attr); - } + pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void *)pNote); + pthread_attr_destroy(&attr); + } - pthread_mutex_unlock(&pNote->taosNoteMutex); + pthread_mutex_unlock(&pNote->mutex); - return pNote->taosNoteFd; + return pNote->fd; } -bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote) -{ - /* - int exist = access(noteName, F_OK); - if (exist != 0) { - return false; - } - */ - - int fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (fd < 0) { - fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); - return true; - } +static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { + int32_t fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd < 0) { + fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); + return true; + } - if (taosLockNote(fd, pNote)) { - taosUnLockNote(fd, pNote); - close(fd); - return false; - } - else { - close(fd); - return true; - } + if (taosLockNote(fd, pNote)) { + taosUnLockNote(fd, pNote); + close(fd); + return false; + } else { + close(fd); + return true; + } } -void taosGetNoteName(char *fn, taosNoteInfo * pNote) -{ - if (pNote->taosNoteFileNum > 1) { - for (int i = 0; i < pNote->taosNoteFileNum; i++) { - char fileName[NOTE_FILE_NAME_LEN]; +static void taosGetNoteName(char *fn, SNoteObj *pNote) { + if (pNote->fileNum > 1) { + for (int32_t i = 0; i < pNote->fileNum; i++) { + char fileName[NOTE_FILE_NAME_LEN]; - sprintf(fileName, "%s%d.0", fn, i); - bool file1open = taosCheckNoteIsOpen(fileName, pNote); + sprintf(fileName, "%s%d.0", fn, i); + bool file1open = taosCheckNoteIsOpen(fileName, pNote); - sprintf(fileName, "%s%d.1", fn, i); - bool file2open = taosCheckNoteIsOpen(fileName, pNote); + sprintf(fileName, "%s%d.1", fn, i); + bool file2open = taosCheckNoteIsOpen(fileName, pNote); - if (!file1open && !file2open) { - sprintf(pNote->taosNoteName, "%s%d", fn, i); - return; - } - } + if (!file1open && !file2open) { + sprintf(pNote->name, "%s%d", fn, i); + return; + } } + } - if (strlen(fn) < NOTE_FILE_NAME_LEN) { - strcpy(pNote->taosNoteName, fn); - } + if (strlen(fn) < NOTE_FILE_NAME_LEN) { + strcpy(pNote->name, fn); + } } -int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote) -{ - char name[NOTE_FILE_NAME_LEN * 2] = "\0"; - struct stat notestat0, notestat1; - int size; - - pNote->taosNoteMaxLines = maxLines; - pNote->taosNoteFileNum = maxNoteNum; - taosGetNoteName(fn, pNote); +static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) { + char name[NOTE_FILE_NAME_LEN * 2] = {0}; + int32_t size; + struct stat logstat0, logstat1; - if (strlen(fn) > NOTE_FILE_NAME_LEN * 2 - 2) { - fprintf(stderr, "the len of file name overflow:%s\n", fn); - return -1; - } + pNote->maxLines = maxLines; + pNote->fileNum = maxNoteNum; + taosGetNoteName(fn, pNote); + if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { strcpy(name, fn); strcat(name, ".0"); + } + bool log0Exist = stat(name, &logstat0) >= 0; - // if none of the note files exist, open 0, if both exists, open the old one - if (stat(name, ¬estat0) < 0) { - pNote->taosNoteFlag = 0; - } else { - strcpy(name, fn); - strcat(name, ".1"); - if (stat(name, ¬estat1) < 0) { - pNote->taosNoteFlag = 1; - } - else { - pNote->taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1; - } - } - - char noteName[NOTE_FILE_NAME_LEN * 2] = "\0"; - sprintf(noteName, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag); - pthread_mutex_init(&pNote->taosNoteMutex, NULL); - - umask(0); - pNote->taosNoteFd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - - if (pNote->taosNoteFd < 0) { - fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - taosLockNote(pNote->taosNoteFd, pNote); - - // only an estimate for number of lines - struct stat filestat; - if (fstat(pNote->taosNoteFd, &filestat) < 0) { - fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - size = (int)filestat.st_size; - pNote->taosNoteLines = size / 60; + if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { + strcpy(name, fn); + strcat(name, ".1"); + } + bool log1Exist = stat(name, &logstat1) >= 0; + + if (!log0Exist && !log1Exist) { + pNote->flag = 0; + } else if (!log1Exist) { + pNote->flag = 0; + } else if (!log0Exist) { + pNote->flag = 1; + } else { + pNote->flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1; + } + + char noteName[NOTE_FILE_NAME_LEN * 2] = {0}; + sprintf(noteName, "%s.%d", pNote->name, pNote->flag); + pthread_mutex_init(&pNote->mutex, NULL); + + umask(0); + pNote->fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + + if (pNote->fd < 0) { + fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); + return -1; + } + taosLockNote(pNote->fd, pNote); + + // only an estimate for number of lines + struct stat filestat; + if (fstat(pNote->fd, &filestat) < 0) { + fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); + return -1; + } + size = (int32_t)filestat.st_size; + pNote->lines = size / 60; + + lseek(pNote->fd, 0, SEEK_END); + + return 0; +} - lseek(pNote->taosNoteFd, 0, SEEK_END); +void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) { + if (pNote->fd <= 0) return; + taosWrite(pNote->fd, buffer, len); - return 0; + if (pNote->maxLines > 0) { + pNote->lines++; + if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote); + } } -void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...) -{ - va_list argpointer; - char buffer[MAX_NOTE_LINE_SIZE+2]; - int len; - struct tm Tm, *ptm; - struct timeval timeSecs; - time_t curTime; - - gettimeofday(&timeSecs, NULL); - curTime = timeSecs.tv_sec; - ptm = localtime_r(&curTime, &Tm); -#ifndef LINUX - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%lld ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec, taosGetPthreadId()); -#else - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %lx ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, - ptm->tm_sec, (int)timeSecs.tv_usec, (unsigned long int)pthread_self()); -#endif - va_start(argpointer, format); - len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); - va_end(argpointer); - - if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2; - - buffer[len++] = '\n'; - buffer[len] = 0; - - if (pNote->taosNoteFd >= 0) { - taosWrite(pNote->taosNoteFd, buffer, (unsigned int)len); - - if (pNote->taosNoteMaxLines > 0) { - pNote->taosNoteLines++; - if ((pNote->taosNoteLines > pNote->taosNoteMaxLines) && (pNote->taosNoteOpenInProgress == 0)) - taosOpenNewNote(pNote); - } - } +void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { + va_list argpointer; + char buffer[MAX_NOTE_LINE_SIZE + 2]; + int32_t len; + struct tm Tm, *ptm; + struct timeval timeSecs; + time_t curTime; + + gettimeofday(&timeSecs, NULL); + curTime = timeSecs.tv_sec; + ptm = localtime_r(&curTime, &Tm); + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); + va_start(argpointer, format); + len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); + va_end(argpointer); + + if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2; + + buffer[len++] = '\n'; + buffer[len] = 0; + + taosNotePrintBuffer(pNote, buffer, len); } -void taosCloseNote(taosNoteInfo * pNote) -{ - taosCloseNoteByFd(pNote->taosNoteFd, pNote); -} +// static void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); } -void taosCloseNoteByFd(int fd, taosNoteInfo * pNote) -{ - if (fd >= 0) { - taosUnLockNote(fd, pNote); - close(fd); - } +static void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) { + if (fd >= 0) { + taosUnLockNote(fd, pNote); + close(fd); + } } diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index 143ce7c47489ac7889fb0d8f047a8591e3d6e280..7caa1a6c37a873bae2741d2139cee808ae5a73b4 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -61,7 +61,7 @@ taos_queue taosOpenQueue() { pthread_mutex_init(&queue->mutex, NULL); - uTrace("queue:%p is openned", queue); + uTrace("queue:%p is opened", queue); return queue; } @@ -230,7 +230,7 @@ taos_qset taosOpenQset() { pthread_mutex_init(&qset->mutex, NULL); tsem_init(&qset->sem, 0, 0); - uTrace("qset:%p is openned", qset); + uTrace("qset:%p is opened", qset); return qset; } @@ -358,15 +358,15 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand if (queue->head) { pNode = queue->head; *pitem = pNode->item; - *type = pNode->type; - *phandle = queue->ahandle; + if (type) *type = pNode->type; + if (phandle) *phandle = queue->ahandle; queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; atomic_sub_fetch_32(&qset->numOfItems, 1); code = 1; - uTrace("item:%p is read out from queue:%p, type:%d items:%d", *pitem, queue, *type, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, type:%d items:%d", *pitem, queue, pNode->type, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 1f83abcb847147da1d9ad0d17d26081176df0c93..7d64bd1f83fb8d235c825057251a5e76e0b96b2a 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -24,22 +24,22 @@ #define TSDB_REF_STATE_DELETED 2 typedef struct SRefNode { - struct SRefNode *prev; // previous node + struct SRefNode *prev; // previous node struct SRefNode *next; // next node - void *p; // pointer to resource protected, + void *p; // pointer to resource protected, int64_t rid; // reference ID int32_t count; // number of references - int removed; // 1: removed + int removed; // 1: removed } SRefNode; typedef struct { SRefNode **nodeList; // array of SRefNode linked list int state; // 0: empty, 1: active; 2: deleted - int rsetId; // refSet ID, global unique + int rsetId; // refSet ID, global unique int64_t rid; // increase by one for each new reference - int max; // mod + int max; // mod int32_t count; // total number of SRefNodes in this set - int64_t *lockedBy; + int64_t *lockedBy; void (*fp)(void *); } SRefSet; @@ -62,9 +62,9 @@ int taosOpenRef(int max, void (*fp)(void *)) SRefSet *pSet; int64_t *lockedBy; int i, rsetId; - + pthread_once(&tsRefModuleInit, taosInitRefModule); - + nodeList = calloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; @@ -79,12 +79,12 @@ int taosOpenRef(int max, void (*fp)(void *)) } pthread_mutex_lock(&tsRefMutex); - + for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; - } + } if (i < TSDB_REF_OBJECTS) { rsetId = tsNextId; @@ -105,7 +105,7 @@ int taosOpenRef(int max, void (*fp)(void *)) free (nodeList); free (lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); - } + } pthread_mutex_unlock(&tsRefMutex); @@ -127,7 +127,7 @@ int taosCloseRef(int rsetId) pthread_mutex_lock(&tsRefMutex); - if (pSet->state == TSDB_REF_STATE_ACTIVE) { + if (pSet->state == TSDB_REF_STATE_ACTIVE) { pSet->state = TSDB_REF_STATE_DELETED; deleted = 1; uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count); @@ -142,7 +142,7 @@ int taosCloseRef(int rsetId) return 0; } -int64_t taosAddRef(int rsetId, void *p) +int64_t taosAddRef(int rsetId, void *p) { int hash; SRefNode *pNode; @@ -163,7 +163,7 @@ int64_t taosAddRef(int rsetId, void *p) terrno = TSDB_CODE_REF_ID_REMOVED; return -1; } - + pNode = calloc(sizeof(SRefNode), 1); if (pNode == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; @@ -173,7 +173,7 @@ int64_t taosAddRef(int rsetId, void *p) rid = atomic_add_fetch_64(&pSet->rid, 1); hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); - + pNode->p = p; pNode->rid = rid; pNode->count = 1; @@ -187,16 +187,16 @@ int64_t taosAddRef(int rsetId, void *p) taosUnlockList(pSet->lockedBy+hash); - return rid; + return rid; } -int taosRemoveRef(int rsetId, int64_t rid) +int taosRemoveRef(int rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid -void *taosAcquireRef(int rsetId, int64_t rid) +void *taosAcquireRef(int rsetId, int64_t rid) { int hash; SRefNode *pNode; @@ -204,7 +204,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) void *p = NULL; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { - uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); + //uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); terrno = TSDB_CODE_REF_INVALID_ID; return NULL; } @@ -223,7 +223,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) terrno = TSDB_CODE_REF_ID_REMOVED; return NULL; } - + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); @@ -233,8 +233,8 @@ void *taosAcquireRef(int rsetId, int64_t rid) if (pNode->rid == rid) { break; } - - pNode = pNode->next; + + pNode = pNode->next; } if (pNode) { @@ -258,7 +258,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) return p; } -int taosReleaseRef(int rsetId, int64_t rid) +int taosReleaseRef(int rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0); } @@ -274,12 +274,13 @@ void *taosIterateRef(int rsetId, int64_t rid) { return NULL; } - if (rid <= 0) { + if (rid < 0) { uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; return NULL; } + void *newP = NULL; pSet = tsRefSetList + rsetId; taosIncRsetCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { @@ -289,52 +290,68 @@ void *taosIterateRef(int rsetId, int64_t rid) { return NULL; } - int hash = 0; - if (rid > 0) { - hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + do { + newP = NULL; + int hash = 0; + if (rid > 0) { + hash = rid % pSet->max; + taosLockList(pSet->lockedBy+hash); - pNode = pSet->nodeList[hash]; - while (pNode) { - if (pNode->rid == rid) break; - pNode = pNode->next; - } + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->rid == rid) break; + pNode = pNode->next; + } - if (pNode == NULL) { - uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); - terrno = TSDB_CODE_REF_NOT_EXIST; - taosUnlockList(pSet->lockedBy+hash); - return NULL; + if (pNode == NULL) { + uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + taosUnlockList(pSet->lockedBy+hash); + taosDecRsetCount(pSet); + return NULL; + } + + // rid is there + pNode = pNode->next; + // check first place + while (pNode) { + if (!pNode->removed) break; + pNode = pNode->next; + } + if (pNode == NULL) { + taosUnlockList(pSet->lockedBy+hash); + hash++; + } } - // rid is there - pNode = pNode->next; - if (pNode == NULL) { - taosUnlockList(pSet->lockedBy+hash); - hash++; + if (pNode == NULL) { + for (; hash < pSet->max; ++hash) { + taosLockList(pSet->lockedBy+hash); + pNode = pSet->nodeList[hash]; + if (pNode) { + // check first place + while (pNode) { + if (!pNode->removed) break; + pNode = pNode->next; + } + if (pNode) break; + } + taosUnlockList(pSet->lockedBy+hash); + } } - } - if (pNode == NULL) { - for (; hash < pSet->max; ++hash) { - taosLockList(pSet->lockedBy+hash); - pNode = pSet->nodeList[hash]; - if (pNode) break; + if (pNode) { + pNode->count++; // acquire it + newP = pNode->p; taosUnlockList(pSet->lockedBy+hash); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); + } else { + uTrace("rsetId:%d the list is over", rsetId); } - } - - void *newP = NULL; - if (pNode) { - pNode->count++; // acquire it - newP = pNode->p; - taosUnlockList(pSet->lockedBy+hash); - uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); - } else { - uTrace("rsetId:%d the list is over", rsetId); - } - if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one + if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one + if (pNode) rid = pNode->rid; + } while (newP && pNode->removed); taosDecRsetCount(pSet); @@ -350,22 +367,22 @@ int taosListRef() { for (int i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; - - if (pSet->state == TSDB_REF_STATE_EMPTY) + + if (pSet->state == TSDB_REF_STATE_EMPTY) continue; uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); for (int j=0; j < pSet->max; ++j) { pNode = pSet->nodeList[j]; - + while (pNode) { uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count); pNode = pNode->next; num++; } - } - } + } + } pthread_mutex_unlock(&tsRefMutex); @@ -397,16 +414,16 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { terrno = TSDB_CODE_REF_ID_REMOVED; return -1; } - + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); - + pNode = pSet->nodeList[hash]; while (pNode) { if (pNode->rid == rid) break; - pNode = pNode->next; + pNode = pNode->next; } if (pNode) { @@ -417,18 +434,18 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (pNode->prev) { pNode->prev->next = pNode->next; } else { - pSet->nodeList[hash] = pNode->next; + pSet->nodeList[hash] = pNode->next; } if (pNode->next) { - pNode->next->prev = pNode->prev; - } + pNode->next->prev = pNode->prev; + } released = 1; } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); } } else { - uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; code = -1; } @@ -437,17 +454,17 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (released) { uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); - (*pSet->fp)(pNode->p); + (*pSet->fp)(pNode->p); free(pNode); taosDecRsetCount(pSet); - } + } return code; } static void taosLockList(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { @@ -457,7 +474,7 @@ static void taosLockList(int64_t *lockedBy) { } static void taosUnlockList(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { assert(false); } @@ -493,5 +510,5 @@ static void taosDecRsetCount(SRefSet *pSet) { } pthread_mutex_unlock(&tsRefMutex); -} +} diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 20bec169546d6514def2453257346d5f94826187..e3798162e8e949d65c6c6382574a43d94a474ee6 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -280,7 +280,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) { tSkipListRLock(pSkipList); if (iter->order == TSDB_ORDER_ASC) { - if (iter->cur == pSkipList->pTail) return false; + // no data in the skip list + if (iter->cur == pSkipList->pTail || iter->next == NULL) { + iter->cur = pSkipList->pTail; + tSkipListUnlock(pSkipList); + return false; + } + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it @@ -292,9 +298,11 @@ bool tSkipListIterNext(SSkipListIterator *iter) { iter->step++; } else { if (iter->cur == pSkipList->pHead) { + iter->cur = pSkipList->pHead; tSkipListUnlock(pSkipList); return false; } + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 2543c8170896825ae2a94349c0218e7d99ce8592..e07587686774cb0fa6c7682daca59b400a9451ec 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -18,6 +18,10 @@ #include "tsocket.h" #include "taoserror.h" +#ifndef SIGPIPE + #define SIGPIPE EPIPE +#endif + int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; @@ -28,21 +32,34 @@ int32_t taosGetFqdn(char *fqdn) { struct addrinfo hints = {0}; struct addrinfo *result = NULL; +#ifdef __APPLE__ + // on macosx, hostname -f has the form of xxx.local + // which will block getaddrinfo for a few seconds if AI_CANONNAME is set + // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return + // immediately + hints.ai_family = AF_INET; +#else // __APPLE__ hints.ai_flags = AI_CANONNAME; +#endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } +#ifdef __APPLE__ + // refer to comments above + strcpy(fqdn, hostname); +#else // __APPLE__ strcpy(fqdn, result->ai_canonname); +#endif // __APPLE__ freeaddrinfo(result); return 0; } -uint32_t taosGetIpFromFqdn(const char *fqdn) { +uint32_t taosGetIpv4FromFqdn(const char *fqdn) { struct addrinfo hints = {0}; - hints.ai_family = AF_UNSPEC; + hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; struct addrinfo *result = NULL; @@ -115,6 +132,10 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nwritten; ptr += nwritten; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); @@ -142,6 +163,10 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nread; ptr += nread; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); @@ -229,7 +254,7 @@ int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; + SOCKET sockFd; int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -239,7 +264,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -268,9 +293,10 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { } SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; + SOCKET sockFd = 0; int32_t ret; struct sockaddr_in serverAddr, clientAddr; + int32_t bufSize = 1024 * 1024; sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); @@ -288,6 +314,18 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie return -1; } + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the send buffer size for TCP socket\n"); + taosCloseSocket(sockFd); + return -1; + } + + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the receive buffer size for TCP socket\n"); + taosCloseSocket(sockFd); + return -1; + } + if (clientIp != 0) { memset((char *)&clientAddr, 0, sizeof(clientAddr)); clientAddr.sin_family = AF_INET; @@ -329,6 +367,8 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { return -1; } +#ifndef __APPLE__ + // all fails on macosx int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -349,6 +389,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { taosCloseSocket(sockFd); return -1; } +#endif // __APPLE__ int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { @@ -381,7 +422,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -424,7 +465,7 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { #define COPY_SIZE 32768 // sendfile shall be used -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { +int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { int64_t leftLen; int32_t readLen, writeLen; char temp[COPY_SIZE]; @@ -455,5 +496,5 @@ int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { leftLen -= readLen; } - return 0; + return (int32_t)len; } diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 0222a6d80a3f36bacde9847b1cc326fe22aa389a..98334497772d60991a553b5d929404ce5e3437e4 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -18,8 +18,9 @@ #include "tsched.h" #include "ttimer.h" #include "tutil.h" +#include "monotonic.h" -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; #define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} #define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} @@ -119,7 +120,7 @@ static void timerDecRef(tmr_obj_t* timer) { } static void lockTimerList(timer_list_t* list) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { @@ -129,7 +130,7 @@ static void lockTimerList(timer_list_t* list) { } static void unlockTimerList(timer_list_t* list) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(list->lockedBy), tid, 0) != tid) { assert(false); tmrError("%" PRId64 " trying to unlock a timer list not locked by current thread.", tid); @@ -186,6 +187,10 @@ static void removeTimer(uintptr_t id) { unlockTimerList(list); } +static int64_t getMonotonicMs(void) { + return (int64_t) getMonotonicUs() / 1000; +} + static void addToWheel(tmr_obj_t* timer, uint32_t delay) { timerAddRef(timer); // select a wheel for the timer, we are not an accurate timer, @@ -201,7 +206,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { time_wheel_t* wheel = wheels + timer->wheel; timer->prev = NULL; - timer->expireAt = taosGetTimestampMs() + delay; + timer->expireAt = getMonotonicMs() + delay; pthread_mutex_lock(&wheel->mutex); @@ -225,10 +230,11 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { } static bool removeFromWheel(tmr_obj_t* timer) { - if (timer->wheel >= tListLen(wheels)) { + uint8_t wheelIdx = timer->wheel; + if (wheelIdx >= tListLen(wheels)) { return false; } - time_wheel_t* wheel = wheels + timer->wheel; + time_wheel_t* wheel = wheels + wheelIdx; bool removed = false; pthread_mutex_lock(&wheel->mutex); @@ -256,7 +262,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { static void processExpiredTimer(void* handle, void* arg) { tmr_obj_t* timer = (tmr_obj_t*)handle; - timer->executedBy = taosGetPthreadId(); + timer->executedBy = taosGetSelfPthreadId(); uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED); if (state == TIMER_STATE_WAITING) { const char* fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] execution start."; @@ -333,7 +339,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle } static void taosTimerLoopFunc(int signo) { - int64_t now = taosGetTimestampMs(); + int64_t now = getMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { // `expried` is a temporary expire list. @@ -405,7 +411,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { return false; } - if (timer->executedBy == taosGetPthreadId()) { + if (timer->executedBy == taosGetSelfPthreadId()) { // taosTmrReset is called in the timer callback, should do nothing in this // case to avoid dead lock. note taosTmrReset must be the last statement // of the callback funtion, will be a bug otherwise. @@ -500,7 +506,7 @@ static void taosTmrModuleInit(void) { pthread_mutex_init(&tmrCtrlMutex, NULL); - int64_t now = taosGetTimestampMs(); + int64_t now = getMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { @@ -531,6 +537,9 @@ static void taosTmrModuleInit(void) { } void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) { + const char* ret = monotonicInit(); + tmrInfo("ttimer monotonic clock source:%s", ret); + pthread_once(&tmrModuleInit, taosTmrModuleInit); pthread_mutex_lock(&tmrCtrlMutex); diff --git a/src/util/src/tworker.c b/src/util/src/tworker.c new file mode 100644 index 0000000000000000000000000000000000000000..8b4053bccd1ce8d9d3f58328d838f4ba5132a100 --- /dev/null +++ b/src/util/src/tworker.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tulog.h" +#include "tqueue.h" +#include "tworker.h" + +int32_t tWorkerInit(SWorkerPool *pPool) { + pPool->qset = taosOpenQset(); + pPool->worker = calloc(sizeof(SWorker), pPool->max); + pthread_mutex_init(&pPool->mutex, NULL); + for (int i = 0; i < pPool->max; ++i) { + SWorker *pWorker = pPool->worker + i; + pWorker->id = i; + pWorker->pPool = pPool; + } + + uInfo("worker:%s is initialized, min:%d max:%d", pPool->name, pPool->min, pPool->max); + return 0; +} + +void tWorkerCleanup(SWorkerPool *pPool) { + for (int i = 0; i < pPool->max; ++i) { + SWorker *pWorker = pPool->worker + i; + if(taosCheckPthreadValid(pWorker->thread)) { + taosQsetThreadResume(pPool->qset); + } + } + + for (int i = 0; i < pPool->max; ++i) { + SWorker *pWorker = pPool->worker + i; + if (taosCheckPthreadValid(pWorker->thread)) { + pthread_join(pWorker->thread, NULL); + } + } + + free(pPool->worker); + taosCloseQset(pPool->qset); + pthread_mutex_destroy(&pPool->mutex); + + uInfo("worker:%s is closed", pPool->name); +} + +void *tWorkerAllocQueue(SWorkerPool *pPool, void *ahandle) { + pthread_mutex_lock(&pPool->mutex); + taos_queue pQueue = taosOpenQueue(); + if (pQueue == NULL) { + pthread_mutex_unlock(&pPool->mutex); + return NULL; + } + + taosAddIntoQset(pPool->qset, pQueue, ahandle); + + // spawn a thread to process queue + if (pPool->num < pPool->max) { + do { + SWorker *pWorker = pPool->worker + pPool->num; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, pPool->workerFp, pWorker) != 0) { + uError("worker:%s:%d failed to create thread to process since %s", pPool->name, pWorker->id, strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + pPool->num++; + uDebug("worker:%s:%d is launched, total:%d", pPool->name, pWorker->id, pPool->num); + } while (pPool->num < pPool->min); + } + + pthread_mutex_unlock(&pPool->mutex); + uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pPool->name, pQueue, ahandle); + + return pQueue; +} + +void tWorkerFreeQueue(SWorkerPool *pPool, void *pQueue) { + taosCloseQueue(pQueue); + uDebug("worker:%s, queue:%p is freed", pPool->name, pQueue); +} diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index de0cdb028b5f9231f57c78fa3411714c93de85c7..3fefbea05ba763dfa856dd52c195d36ce70ccd91 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -1,6 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) @@ -9,7 +10,5 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) - ADD_LIBRARY(vnode ${SRC}) - TARGET_LINK_LIBRARIES(vnode tsdb tcq) -ENDIF () +ADD_LIBRARY(vnode ${SRC}) +TARGET_LINK_LIBRARIES(vnode tsdb tcq common) diff --git a/src/vnode/inc/vnodeCfg.h b/src/vnode/inc/vnodeCfg.h index c5887fef5d53ca32c215a5f2da9d182386ebc4f0..ba148c07c1e4f9451232c706fb79c0af0cad9746 100644 --- a/src/vnode/inc/vnodeCfg.h +++ b/src/vnode/inc/vnodeCfg.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "vnodeInt.h" int32_t vnodeReadCfg(SVnodeObj *pVnode); int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg); diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index e468c2e83e1560b0711e42d827da9604473f47b0..3ec77bbc122341935a804448eca61258c634cf58 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -19,11 +19,11 @@ #ifdef __cplusplus extern "C" { #endif - #include "tlog.h" #include "tsync.h" -#include "twal.h" #include "tcq.h" +#include "tsdb.h" +#include "vnode.h" extern int32_t vDebugFlag; @@ -35,39 +35,42 @@ extern int32_t vDebugFlag; #define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }} typedef struct { - int32_t vgId; // global vnode group ID - int32_t refCount; // reference count - int32_t queuedWMsg; - int32_t queuedRMsg; - int32_t flowctrlLevel; - int8_t status; - int8_t role; - int8_t accessState; - int8_t isFull; - int8_t isCommiting; - uint64_t version; // current version - uint64_t fversion; // version on saved data file - void *wqueue; - void *rqueue; - void *wal; - void *tsdb; - int64_t sync; - void *events; - void *cq; // continuous query - int32_t cfgVersion; - STsdbCfg tsdbCfg; - SSyncCfg syncCfg; - SWalCfg walCfg; - void *qMgmt; - char *rootDir; - tsem_t sem; - int8_t dropped; - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + int32_t vgId; // global vnode group ID + int32_t refCount; // reference count + int32_t queuedWMsg; + int32_t queuedRMsg; + int32_t flowctrlLevel; + int8_t status; + int8_t role; + int8_t accessState; + int8_t isFull; + int8_t isCommiting; + int8_t dbReplica; + int8_t dropped; + int8_t reserved; + uint64_t version; // current version + uint64_t cversion; // version while commit start + uint64_t fversion; // version on saved data file + void * wqueue; // write queue + void * qqueue; // read query queue + void * fqueue; // read fetch/cancel queue + void * wal; + void * tsdb; + int64_t sync; + void * events; + void * cq; // continuous query + int32_t dbCfgVersion; + int32_t vgCfgVersion; + STsdbCfg tsdbCfg; + SSyncCfg syncCfg; + SWalCfg walCfg; + void * qMgmt; + char * rootDir; + tsem_t sem; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + pthread_mutex_t statusMutex; } SVnodeObj; -void vnodeInitWriteFp(void); -void vnodeInitReadFp(void); - #ifdef __cplusplus } #endif diff --git a/src/vnode/inc/vnodeMain.h b/src/vnode/inc/vnodeMain.h new file mode 100644 index 0000000000000000000000000000000000000000..73591bc10d97dfbe519cf9e5c1f73a96c8fa0854 --- /dev/null +++ b/src/vnode/inc/vnodeMain.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_MAIN_H +#define TDENGINE_VNODE_MAIN_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); +int32_t vnodeDrop(int32_t vgId); +int32_t vnodeOpen(int32_t vgId); +int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); +int32_t vnodeClose(int32_t vgId); +void vnodeCleanUp(SVnodeObj *pVnode); +void vnodeDestroy(SVnodeObj *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/vnode/inc/vnodeMgmt.h b/src/vnode/inc/vnodeMgmt.h new file mode 100644 index 0000000000000000000000000000000000000000..5a7e7456195f2f8a21f2c99b4d50761c83729acd --- /dev/null +++ b/src/vnode/inc/vnodeMgmt.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_MGMT_H +#define TDENGINE_VNODE_MGMT_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeInitMgmt(); +void vnodeCleanupMgmt(); + +void* vnodeAcquire(int32_t vgId); +void vnodeRelease(void *pVnode); +void* vnodeGetWal(void *pVnode); + +int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); +void vnodeBuildStatusMsg(void *pStatus); +void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); + +void vnodeAddIntoHash(SVnodeObj* pVnode); +void vnodeRemoveFromHash(SVnodeObj * pVnode); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/common/inc/tsystem.h b/src/vnode/inc/vnodeRead.h similarity index 62% rename from src/common/inc/tsystem.h rename to src/vnode/inc/vnodeRead.h index 93d305e49c6707e7c567a2fbca94b652067f421a..0e9655f837e311fd8d7d8779ce915e5b152be288 100644 --- a/src/common/inc/tsystem.h +++ b/src/vnode/inc/vnodeRead.h @@ -13,23 +13,21 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSYSTEM_H -#define TDENGINE_TSYSTEM_H +#ifndef TDENGINE_VNODE_READ_H +#define TDENGINE_VNODE_READ_H #ifdef __cplusplus extern "C" { #endif +#include "vnodeInt.h" -bool taosGetSysMemory(float *memoryUsedMB); -bool taosGetProcMemory(float *memoryUsedMB); -bool taosGetDisk(); -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); -bool taosGetBandSpeed(float *bandSpeedKb); -bool taosGetProcIO(float *readKB, float *writeKB); -void taosGetSystemInfo(); -void taosPrintOsInfo(); -void taosKillSystem(); -void taosSetCoreDump(); +int32_t vnodeInitRead(void); +void vnodeCleanupRead(void); + +int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam); +void vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead); +int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); +void vnodeWaitReadCompleted(SVnodeObj *pVnode); #ifdef __cplusplus } diff --git a/src/vnode/inc/vnodeStatus.h b/src/vnode/inc/vnodeStatus.h new file mode 100644 index 0000000000000000000000000000000000000000..910a6d71b201fcdc9fbc1daa99fb57d0227d6093 --- /dev/null +++ b/src/vnode/inc/vnodeStatus.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_STATUS_H +#define TDENGINE_VNODE_STATUS_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +typedef enum _VN_STATUS { + TAOS_VN_STATUS_INIT = 0, + TAOS_VN_STATUS_READY = 1, + TAOS_VN_STATUS_CLOSING = 2, + TAOS_VN_STATUS_UPDATING = 3, + TAOS_VN_STATUS_RESET = 4, +} EVnodeStatus; + +bool vnodeSetInitStatus(SVnodeObj* pVnode); +bool vnodeSetReadyStatus(SVnodeObj* pVnode); +bool vnodeSetClosingStatus(SVnodeObj* pVnode); +bool vnodeSetUpdatingStatus(SVnodeObj* pVnode); +bool vnodeSetResetStatus(SVnodeObj* pVnode); + +bool vnodeInInitStatus(SVnodeObj* pVnode); +bool vnodeInReadyStatus(SVnodeObj* pVnode); +bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode); +bool vnodeInClosingStatus(SVnodeObj* pVnode); +bool vnodeInResetStatus(SVnodeObj* pVnode); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/vnode/inc/vnodeSync.h b/src/vnode/inc/vnodeSync.h new file mode 100644 index 0000000000000000000000000000000000000000..c9ac25c2274d81cd08c52a77cd3cc76a27c7a0d5 --- /dev/null +++ b/src/vnode/inc/vnodeSync.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_SYNC_H +#define TDENGINE_VNODE_SYNC_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver); +int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); +void vnodeNotifyRole(int32_t vgId, int8_t role); +void vnodeCtrlFlow(int32_t vgId, int32_t level); +void vnodeStartSyncFile(int32_t vgId); +void vnodeStopSyncFile(int32_t vgId, uint64_t fversion); +void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); +int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); +int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); + +void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/vnode/inc/vnodeVersion.h b/src/vnode/inc/vnodeVersion.h index 1d086cb21fdab0247038c7f3d32d89b38c19d871..913e3915ab3b911746440be9eab2c7f2e05dce3d 100644 --- a/src/vnode/inc/vnodeVersion.h +++ b/src/vnode/inc/vnodeVersion.h @@ -19,6 +19,7 @@ #ifdef __cplusplus extern "C" { #endif +#include "vnodeInt.h" int32_t vnodeReadVersion(SVnodeObj *pVnode); int32_t vnodeSaveVersion(SVnodeObj *pVnode); diff --git a/src/vnode/inc/vnodeWorker.h b/src/vnode/inc/vnodeWorker.h new file mode 100644 index 0000000000000000000000000000000000000000..01d9d42900ef7e0c56bd396f5c698a43c4b501f8 --- /dev/null +++ b/src/vnode/inc/vnodeWorker.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_WORKER_H +#define TDENGINE_VNODE_WORKER_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeInitMWorker(); +void vnodeCleanupMWorker(); +int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode); +int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/vnode/inc/vnodeWrite.h b/src/vnode/inc/vnodeWrite.h new file mode 100644 index 0000000000000000000000000000000000000000..e996bc0b06f4563b24e5ebdfc90420b9e67df003 --- /dev/null +++ b/src/vnode/inc/vnodeWrite.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_WRITE_H +#define TDENGINE_VNODE_WRITE_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeInitWrite(void); +void vnodeCleanupWrite(void); + +int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg); +void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite); +int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet); +void vnodeWaitWriteCompleted(SVnodeObj *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index 2d56157328714a5ae4b8ac05abe3ac4468e361cb..03f2b11eec239ad469faec66ddcbd60282e0409b 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -15,18 +15,15 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosmsg.h" -#include "taoserror.h" #include "cJSON.h" #include "tglobal.h" -#include "tsdb.h" #include "dnode.h" -#include "vnodeInt.h" #include "vnodeCfg.h" static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db)); - pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion; + pVnode->dbCfgVersion = vnodeMsg->cfg.dbCfgVersion; + pVnode->vgCfgVersion = vnodeMsg->cfg.vgCfgVersion; pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize; pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks; pVnode->tsdbCfg.daysPerFile = vnodeMsg->cfg.daysPerFile; @@ -37,11 +34,14 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->tsdbCfg.update = vnodeMsg->cfg.update; + pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow; pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP; - pVnode->syncCfg.replica = vnodeMsg->cfg.replications; + pVnode->syncCfg.replica = vnodeMsg->cfg.vgReplica; pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum; + pVnode->dbReplica = vnodeMsg->cfg.dbReplica; for (int i = 0; i < pVnode->syncCfg.replica; ++i) { SVnodeDesc *node = &vnodeMsg->nodes[i]; @@ -79,7 +79,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_VCFG_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VCFG_ERROR; @@ -99,68 +99,76 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } tstrncpy(vnodeMsg.db, db->valuestring, sizeof(vnodeMsg.db)); - cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); - if (!cfgVersion || cfgVersion->type != cJSON_Number) { + cJSON *dbCfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); + if (!dbCfgVersion || dbCfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.cfgVersion = cfgVersion->valueint; + vnodeMsg.cfg.dbCfgVersion = (int32_t)dbCfgVersion->valueint; + + cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion"); + if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file); + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.vgCfgVersion = (int32_t)vgCfgVersion->valueint; + } cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint; + vnodeMsg.cfg.cacheBlockSize = (int32_t)cacheBlockSize->valueint; cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); if (!totalBlocks || totalBlocks->type != cJSON_Number) { vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.totalBlocks = totalBlocks->valueint; + vnodeMsg.cfg.totalBlocks = (int32_t)totalBlocks->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysPerFile = daysPerFile->valueint; + vnodeMsg.cfg.daysPerFile = (int32_t)daysPerFile->valueint; cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); if (!daysToKeep || daysToKeep->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep = daysToKeep->valueint; + vnodeMsg.cfg.daysToKeep = (int32_t)daysToKeep->valueint; cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint; + vnodeMsg.cfg.daysToKeep1 = (int32_t)daysToKeep1->valueint; cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint; + vnodeMsg.cfg.daysToKeep2 = (int32_t)daysToKeep2->valueint; cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; + vnodeMsg.cfg.minRowsPerFileBlock = (int32_t)minRowsPerFileBlock->valueint; cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; + vnodeMsg.cfg.maxRowsPerFileBlock = (int32_t)maxRowsPerFileBlock->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { @@ -188,7 +196,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint; + vnodeMsg.cfg.fsyncPeriod = (int32_t)fsyncPeriod->valueint; cJSON *wals = cJSON_GetObjectItem(root, "wals"); if (!wals || wals->type != cJSON_Number) { @@ -197,12 +205,21 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.wals = (int8_t)wals->valueint; - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { + cJSON *vgReplica = cJSON_GetObjectItem(root, "replica"); + if (!vgReplica || vgReplica->type != cJSON_Number) { vError("vgId:%d, failed to read %s, replica not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.replications = (int8_t)replica->valueint; + vnodeMsg.cfg.vgReplica = (int8_t)vgReplica->valueint; + + cJSON *dbReplica = cJSON_GetObjectItem(root, "dbReplica"); + if (!dbReplica || dbReplica->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file); + vnodeMsg.cfg.dbReplica = vnodeMsg.cfg.vgReplica; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.dbReplica = (int8_t)dbReplica->valueint; + } cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); if (!quorum || quorum->type != cJSON_Number) { @@ -211,6 +228,24 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + cJSON *update = cJSON_GetObjectItem(root, "update"); + if (!update || update->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, update not found", pVnode->vgId, file); + vnodeMsg.cfg.update = 0; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.update = (int8_t)update->valueint; + } + + cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow"); + if (!cacheLastRow || cacheLastRow->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file); + vnodeMsg.cfg.cacheLastRow = 0; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.cacheLastRow = (int8_t)cacheLastRow->valueint; + } + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file); @@ -218,7 +253,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } int size = cJSON_GetArraySize(nodeInfos); - if (size != vnodeMsg.cfg.replications) { + if (size != vnodeMsg.cfg.vgReplica) { vError("vgId:%d, failed to read %s, nodeInfos size not matched", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } @@ -233,7 +268,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - node->nodeId = nodeId->valueint; + node->nodeId = (int32_t)nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { @@ -242,9 +277,8 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN); - if (!nodeChanged) { - nodeChanged = dnodeCheckEpChanged(node->nodeId, node->nodeEp); - } + bool changed = dnodeCheckEpChanged(node->nodeId, node->nodeEp); + if (changed) nodeChanged = changed; } ret = TSDB_CODE_SUCCESS; @@ -283,7 +317,8 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pMsg->db); - len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.cfgVersion); + len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.dbCfgVersion); + len += snprintf(content + len, maxLen - len, " \"vgCfgVersion\": %d,\n", pMsg->cfg.vgCfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pMsg->cfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pMsg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pMsg->cfg.daysPerFile); @@ -296,16 +331,19 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pMsg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pMsg->cfg.walLevel); len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pMsg->cfg.fsyncPeriod); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.replications); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.vgReplica); + len += snprintf(content + len, maxLen - len, " \"dbReplica\": %d,\n", pMsg->cfg.dbReplica); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"update\": %d,\n", pMsg->cfg.update); + len += snprintf(content + len, maxLen - len, " \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < pMsg->cfg.replications; i++) { + for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) { SVnodeDesc *node = &pMsg->nodes[i]; dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL); len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId); len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp); - if (i < pMsg->cfg.replications - 1) { + if (i < pMsg->cfg.vgReplica - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); @@ -314,7 +352,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index b516c9d90e126d68ca502af576d6f400dedd175e..ac9536d243dbd2613d42c8b9a5b054ce862df061 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -18,77 +18,18 @@ #include "taoserror.h" #include "taosmsg.h" #include "tglobal.h" -#include "trpc.h" -#include "tsdb.h" -#include "tutil.h" -#include "vnode.h" -#include "vnodeInt.h" +#include "tfs.h" #include "query.h" #include "dnode.h" #include "vnodeCfg.h" +#include "vnodeStatus.h" +#include "vnodeSync.h" #include "vnodeVersion.h" +#include "vnodeMgmt.h" +#include "vnodeWorker.h" +#include "vnodeMain.h" -static SHashObj*tsVnodesHash; -static void vnodeCleanUp(SVnodeObj *pVnode); -static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); -static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); -static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); -static void vnodeNotifyRole(int32_t vgId, int8_t role); -static void vnodeCtrlFlow(int32_t vgId, int32_t level); -static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); -static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); -static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); -static int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); - -#ifndef _SYNC -int64_t syncStart(const SSyncInfo *info) { return NULL; } -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype) { return 0; } -void syncStop(int64_t rid) {} -int32_t syncReconfig(int64_t rid, const SSyncCfg *cfg) { return 0; } -int32_t syncGetNodesRole(int64_t rid, SNodesRole *cfg) { return 0; } -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {} -#endif - -char* vnodeStatus[] = { - "init", - "ready", - "closing", - "updating", - "reset" -}; - -int32_t vnodeInitResources() { - int32_t code = syncInit(); - if (code != 0) return code; - - vnodeInitWriteFp(); - vnodeInitReadFp(); - - tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (tsVnodesHash == NULL) { - vError("failed to init vnode list"); - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - if (tsdbInitCommitQueue(tsNumOfCommitThreads) < 0) { - vError("failed to init vnode commit queue"); - return terrno; - } - - return TSDB_CODE_SUCCESS; -} - -void vnodeCleanupResources() { - tsdbDestroyCommitQueue(); - - if (tsVnodesHash != NULL) { - vDebug("vnode list is cleanup"); - taosHashCleanup(tsVnodesHash); - tsVnodesHash = NULL; - } - - syncCleanUp(); -} +static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { int32_t code; @@ -100,32 +41,19 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_SUCCESS; } - if (mkdir(tsVnodeDir, 0755) != 0 && errno != EEXIST) { - vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), tsVnodeDir); - if (errno == EACCES) { - return TSDB_CODE_VND_NO_DISK_PERMISSIONS; - } else if (errno == ENOSPC) { - return TSDB_CODE_VND_NO_DISKSPACE; - } else if (errno == ENOENT) { - return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR; - } else { - return TSDB_CODE_VND_INIT_FAILED; - } + if (tfsMkdir("vnode") < 0) { + vError("vgId:%d, failed to create vnode dir, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); + return terrno; } char rootDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId); - if (mkdir(rootDir, 0755) != 0 && errno != EEXIST) { - vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir); - if (errno == EACCES) { - return TSDB_CODE_VND_NO_DISK_PERMISSIONS; - } else if (errno == ENOSPC) { - return TSDB_CODE_VND_NO_DISKSPACE; - } else if (errno == ENOENT) { - return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR; - } else { - return TSDB_CODE_VND_INIT_FAILED; - } + + char vnodeDir[TSDB_FILENAME_LEN] = "\0"; + snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d", pVnodeCfg->cfg.vgId); + if (tfsMkdir(vnodeDir) < 0) { + vError("vgId:%d, failed to create vnode dir %s, reason:%s", pVnodeCfg->cfg.vgId, vnodeDir, strerror(errno)); + return terrno; } code = vnodeWriteCfg(pVnodeCfg); @@ -134,28 +62,31 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return code; } - STsdbCfg tsdbCfg = {0}; - tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; - tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; - tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; - tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; - tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; - tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; - tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; - tsdbCfg.precision = pVnodeCfg->cfg.precision; - tsdbCfg.compression = pVnodeCfg->cfg.compression; - tsdbCfg.update = pVnodeCfg->cfg.update; - - char tsdbDir[TSDB_FILENAME_LEN] = {0}; - sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); - if (tsdbCreateRepo(tsdbDir, &tsdbCfg) < 0) { + // STsdbCfg tsdbCfg = {0}; + // tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; + // tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; + // tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; + // tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; + // tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; + // tsdbCfg.keep1 = pVnodeCfg->cfg.daysToKeep1; + // tsdbCfg.keep2 = pVnodeCfg->cfg.daysToKeep2; + // tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; + // tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; + // tsdbCfg.precision = pVnodeCfg->cfg.precision; + // tsdbCfg.compression = pVnodeCfg->cfg.compression; + // tsdbCfg.update = pVnodeCfg->cfg.update; + // tsdbCfg.cacheLastRow = pVnodeCfg->cfg.cacheLastRow; + + // char tsdbDir[TSDB_FILENAME_LEN] = {0}; + // sprintf(tsdbDir, "vnode/vnode%d/tsdb", pVnodeCfg->cfg.vgId); + if (tsdbCreateRepo(pVnodeCfg->cfg.vgId) < 0) { vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); return TSDB_CODE_VND_INIT_FAILED; } vInfo("vgId:%d, vnode dir is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, pVnodeCfg->cfg.fsyncPeriod); - code = vnodeOpen(pVnodeCfg->cfg.vgId, rootDir); + code = vnodeOpen(pVnodeCfg->cfg.vgId); return code; } @@ -170,90 +101,109 @@ int32_t vnodeDrop(int32_t vgId) { vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; + // remove from hash, so new messages wont be consumed + vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); - vnodeCleanUp(pVnode); + vnodeCleanupInMWorker(pVnode); return TSDB_CODE_SUCCESS; } -int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { - SVnodeObj *pVnode = vparam; - - // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS - // cfgVersion can be corrected by status msg - if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_UPDATING) != TAOS_VN_STATUS_READY) { - vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId); - return TSDB_CODE_SUCCESS; - } - +static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) { int32_t code = vnodeWriteCfg(pVnodeCfg); if (code != TSDB_CODE_SUCCESS) { - pVnode->status = TAOS_VN_STATUS_READY; return code; } code = vnodeReadCfg(pVnode); if (code != TSDB_CODE_SUCCESS) { - pVnode->status = TAOS_VN_STATUS_READY; return code; } code = walAlter(pVnode->wal, &pVnode->walCfg); if (code != TSDB_CODE_SUCCESS) { - pVnode->status = TAOS_VN_STATUS_READY; return code; } code = syncReconfig(pVnode->sync, &pVnode->syncCfg); if (code != TSDB_CODE_SUCCESS) { - pVnode->status = TAOS_VN_STATUS_READY; return code; - } + } if (pVnode->tsdb) { code = tsdbConfigRepo(pVnode->tsdb, &pVnode->tsdbCfg); if (code != TSDB_CODE_SUCCESS) { - pVnode->status = TAOS_VN_STATUS_READY; - return code; + return code; } } - pVnode->status = TAOS_VN_STATUS_READY; - vDebug("vgId:%d, vnode is altered", pVnode->vgId); + return 0; +} - return TSDB_CODE_SUCCESS; +int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { + SVnodeObj *pVnode = vparam; + if (pVnode->dbCfgVersion == pVnodeCfg->cfg.dbCfgVersion && pVnode->vgCfgVersion == pVnodeCfg->cfg.vgCfgVersion) { + vDebug("vgId:%d, dbCfgVersion:%d and vgCfgVersion:%d not change", pVnode->vgId, pVnode->dbCfgVersion, + pVnode->vgCfgVersion); + return TSDB_CODE_SUCCESS; + } + + // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS + // dbCfgVersion can be corrected by status msg + if (!vnodeSetUpdatingStatus(pVnode)) { + vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId); + return TSDB_CODE_SUCCESS; + } + + int32_t code = vnodeAlterImp(pVnode, pVnodeCfg); + vnodeSetReadyStatus(pVnode); + + if (code != 0) { + vError("vgId:%d, failed to alter vnode, code:0x%x", pVnode->vgId, code); + } else { + vDebug("vgId:%d, vnode is altered", pVnode->vgId); + } + + return code; } -int32_t vnodeOpen(int32_t vnode, char *rootDir) { - char temp[TSDB_FILENAME_LEN]; +int32_t vnodeOpen(int32_t vgId) { + char temp[TSDB_FILENAME_LEN * 3]; + char rootDir[TSDB_FILENAME_LEN * 2]; + snprintf(rootDir, TSDB_FILENAME_LEN * 2, "%s/vnode%d", tsVnodeDir, vgId); SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1); if (pVnode == NULL) { - vError("vgId:%d, failed to open vnode since no enough memory", vnode); + vError("vgId:%d, failed to open vnode since no enough memory", vgId); return TAOS_SYSTEM_ERROR(errno); } atomic_add_fetch_32(&pVnode->refCount, 1); - pVnode->vgId = vnode; - pVnode->status = TAOS_VN_STATUS_INIT; + pVnode->vgId = vgId; pVnode->fversion = 0; pVnode->version = 0; pVnode->tsdbCfg.tsdbId = pVnode->vgId; pVnode->rootDir = strdup(rootDir); pVnode->accessState = TSDB_VN_ALL_ACCCESS; tsem_init(&pVnode->sem, 0, 0); + pthread_mutex_init(&pVnode->statusMutex, NULL); + vnodeSetInitStatus(pVnode); + + tsdbIncCommitRef(pVnode->vgId); int32_t code = vnodeReadCfg(pVnode); if (code != TSDB_CODE_SUCCESS) { + vError("vgId:%d, failed to read config file, set cfgVersion to 0", pVnode->vgId); vnodeCleanUp(pVnode); - return code; + return 0; } code = vnodeReadVersion(pVnode); if (code != TSDB_CODE_SUCCESS) { - vError("vgId:%d, failed to read version, generate it from data file", pVnode->vgId); - // Allow vnode start even when read version fails, set version as walVersion or zero + pVnode->version = 0; + vError("vgId:%d, failed to read file version, generate it from data file", pVnode->vgId); + // Allow vnode start even when read file version fails, set file version as wal version or zero // vnodeCleanUp(pVnode); // return code; } @@ -261,8 +211,9 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->fversion = pVnode->version; pVnode->wqueue = dnodeAllocVWriteQueue(pVnode); - pVnode->rqueue = dnodeAllocVReadQueue(pVnode); - if (pVnode->wqueue == NULL || pVnode->rqueue == NULL) { + pVnode->qqueue = dnodeAllocVQueryQueue(pVnode); + pVnode->fqueue = dnodeAllocVFetchQueue(pVnode); + if (pVnode->wqueue == NULL || pVnode->qqueue == NULL || pVnode->fqueue == NULL) { vnodeCleanUp(pVnode); return terrno; } @@ -272,7 +223,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { sprintf(cqCfg.user, "_root"); strcpy(cqCfg.pass, tsInternalPass); strcpy(cqCfg.db, pVnode->db); - cqCfg.vgId = vnode; + cqCfg.vgId = vgId; cqCfg.cqWrite = vnodeWriteToCache; pVnode->cq = cqOpen(pVnode, &cqCfg); if (pVnode->cq == NULL) { @@ -287,14 +238,13 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { appH.cqH = pVnode->cq; appH.cqCreateFunc = cqCreate; appH.cqDropFunc = cqDrop; - sprintf(temp, "%s/tsdb", rootDir); terrno = 0; - pVnode->tsdb = tsdbOpenRepo(temp, &appH); + pVnode->tsdb = tsdbOpenRepo(&(pVnode->tsdbCfg), &appH); if (pVnode->tsdb == NULL) { vnodeCleanUp(pVnode); return terrno; - } else if (terrno != TSDB_CODE_SUCCESS) { + } else if (tsdbGetState(pVnode->tsdb) != TSDB_STATE_OK) { vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); if (pVnode->syncCfg.replica <= 1) { @@ -339,38 +289,36 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->events = NULL; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); - tsdbIncCommitRef(pVnode->vgId); - - taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + vnodeAddIntoHash(pVnode); + SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; syncInfo.version = pVnode->version; syncInfo.syncCfg = pVnode->syncCfg; - sprintf(syncInfo.path, "%s", rootDir); - syncInfo.getWalInfo = vnodeGetWalInfo; - syncInfo.getFileInfo = vnodeGetFileInfo; - syncInfo.writeToCache = vnodeWriteToCache; + tstrncpy(syncInfo.path, rootDir, TSDB_FILENAME_LEN); + syncInfo.getWalInfoFp = vnodeGetWalInfo; + syncInfo.writeToCacheFp = vnodeWriteToCache; syncInfo.confirmForward = vnodeConfirmForard; - syncInfo.notifyRole = vnodeNotifyRole; - syncInfo.notifyFlowCtrl = vnodeCtrlFlow; - syncInfo.notifyFileSynced = vnodeNotifyFileSynced; - syncInfo.getVersion = vnodeGetVersion; + syncInfo.notifyRoleFp = vnodeNotifyRole; + syncInfo.notifyFlowCtrlFp = vnodeCtrlFlow; + syncInfo.startSyncFileFp = vnodeStartSyncFile; + syncInfo.stopSyncFileFp = vnodeStopSyncFile; + syncInfo.getVersionFp = vnodeGetVersion; + syncInfo.sendFileFp = tsdbSyncSend; + syncInfo.recvFileFp = tsdbSyncRecv; + syncInfo.pTsdb = pVnode->tsdb; pVnode->sync = syncStart(&syncInfo); -#ifndef _SYNC - pVnode->role = TAOS_SYNC_ROLE_MASTER; -#else if (pVnode->sync <= 0) { - vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, + vError("vgId:%d, failed to open sync, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); - vnodeRelease(pVnode); + vnodeRemoveFromHash(pVnode); vnodeCleanUp(pVnode); return terrno; } -#endif - pVnode->status = TAOS_VN_STATUS_READY; + vnodeSetReadyStatus(pVnode); return TSDB_CODE_SUCCESS; } @@ -379,31 +327,17 @@ int32_t vnodeClose(int32_t vgId) { if (pVnode == NULL) return 0; vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); + vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanUp(pVnode); return 0; } -void vnodeRelease(void *vparam) { - if (vparam == NULL) return; - SVnodeObj *pVnode = vparam; - int32_t code = 0; - int32_t vgId = pVnode->vgId; - - int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode); - assert(refCount >= 0); - - if (refCount > 0) { - if (pVnode->status == TAOS_VN_STATUS_RESET && refCount <= 3) { - tsem_post(&pVnode->sem); - } - return; - } - - vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode); - +void vnodeDestroy(SVnodeObj *pVnode) { + int32_t code = 0; + int32_t vgId = pVnode->vgId; + if (pVnode->qMgmt) { qCleanupQueryMgmt(pVnode->qMgmt); pVnode->qMgmt = NULL; @@ -440,9 +374,14 @@ void vnodeRelease(void *vparam) { pVnode->wqueue = NULL; } - if (pVnode->rqueue) { - dnodeFreeVReadQueue(pVnode->rqueue); - pVnode->rqueue = NULL; + if (pVnode->qqueue) { + dnodeFreeVQueryQueue(pVnode->qqueue); + pVnode->qqueue = NULL; + } + + if (pVnode->fqueue) { + dnodeFreeVFetchQueue(pVnode->fqueue); + pVnode->fqueue = NULL; } tfree(pVnode->rootDir); @@ -450,141 +389,30 @@ void vnodeRelease(void *vparam) { if (pVnode->dropped) { char rootDir[TSDB_FILENAME_LEN] = {0}; char newDir[TSDB_FILENAME_LEN] = {0}; - sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); - sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId); + sprintf(rootDir, "%s/vnode%d", "vnode", vgId); + sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); if (0 == tsEnableVnodeBak) { vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId); } else { - taosRemoveDir(newDir); - taosRename(rootDir, newDir); + tfsRmdir(newDir); + tfsRename(rootDir, newDir); } - taosRemoveDir(rootDir); + tfsRmdir(rootDir); dnodeSendStatusMsgToMnode(); } tsem_destroy(&pVnode->sem); + pthread_mutex_destroy(&pVnode->statusMutex); free(pVnode); tsdbDecCommitRef(vgId); - - int32_t count = taosHashGetSize(tsVnodesHash); - vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); -} - -static void vnodeIncRef(void *ptNode) { - assert(ptNode != NULL); - - SVnodeObj **ppVnode = (SVnodeObj **)ptNode; - assert(ppVnode); - assert(*ppVnode); - - SVnodeObj *pVnode = *ppVnode; - atomic_add_fetch_32(&pVnode->refCount, 1); - vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); -} - -void *vnodeAcquire(int32_t vgId) { - SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); - - if (ppVnode == NULL || *ppVnode == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - vDebug("vgId:%d, not exist", vgId); - return NULL; - } - - return *ppVnode; } -void *vnodeGetWal(void *pVnode) { - return ((SVnodeObj *)pVnode)->wal; -} - -static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { - int64_t totalStorage = 0; - int64_t compStorage = 0; - int64_t pointsWritten = 0; - - if (pVnode->status != TAOS_VN_STATUS_READY) return; - if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; - - if (pVnode->tsdb) { - tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); - } - - SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; - pLoad->vgId = htonl(pVnode->vgId); - pLoad->cfgVersion = htonl(pVnode->cfgVersion); - pLoad->totalStorage = htobe64(totalStorage); - pLoad->compStorage = htobe64(compStorage); - pLoad->pointsWritten = htobe64(pointsWritten); - pLoad->status = pVnode->status; - pLoad->role = pVnode->role; - pLoad->replica = pVnode->syncCfg.replica; -} - -int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - void *pIter = taosHashIterate(tsVnodesHash, NULL); - while (pIter) { - SVnodeObj **pVnode = pIter; - if (*pVnode) { - - (*numOfVnodes)++; - if (*numOfVnodes >= TSDB_MAX_VNODES) { - vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES); - continue; - } else { - vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId; - } - - } - - pIter = taosHashIterate(tsVnodesHash, pIter); - } - return TSDB_CODE_SUCCESS; -} - -void vnodeBuildStatusMsg(void *param) { - SStatusMsg *pStatus = param; - - void *pIter = taosHashIterate(tsVnodesHash, NULL); - while (pIter) { - SVnodeObj **pVnode = pIter; - if (*pVnode) { - vnodeBuildVloadMsg(*pVnode, pStatus); - } - pIter = taosHashIterate(tsVnodesHash, pIter); - } -} +void vnodeCleanUp(SVnodeObj *pVnode) { + vDebug("vgId:%d, vnode will cleanup, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); -void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { - for (int32_t i = 0; i < numOfVnodes; ++i) { - pAccess[i].vgId = htonl(pAccess[i].vgId); - SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId); - if (pVnode != NULL) { - pVnode->accessState = pAccess[i].accessState; - if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) { - vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState); - } - vnodeRelease(pVnode); - } - } -} - -static void vnodeCleanUp(SVnodeObj *pVnode) { - // remove from hash, so new messages wont be consumed - taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t)); - - if (pVnode->status != TAOS_VN_STATUS_INIT) { - // it may be in updateing or reset state, then it shall wait - int32_t i = 0; - while (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_CLOSING) != - TAOS_VN_STATUS_READY) { - if (++i % 1000 == 0) { - sched_yield(); - } - } - } + vnodeSetClosingStatus(pVnode); // stop replication module if (pVnode->sync > 0) { @@ -593,10 +421,7 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { syncStop(sync); } - vDebug("vgId:%d, vnode will cleanup, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - - // release local resources only after cutting off outside connections - qQueryMgmtNotifyClosed(pVnode->qMgmt); + vDebug("vgId:%d, vnode is cleaned, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); vnodeRelease(pVnode); } @@ -613,19 +438,20 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { if (status == TSDB_STATUS_COMMIT_START) { pVnode->isCommiting = 1; - pVnode->fversion = pVnode->version; + pVnode->cversion = pVnode->version; vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); - if (pVnode->status != TAOS_VN_STATUS_INIT) { + if (!vnodeInInitStatus(pVnode)) { return walRenew(pVnode->wal); } return 0; } if (status == TSDB_STATUS_COMMIT_OVER) { - vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); pVnode->isCommiting = 0; pVnode->isFull = 0; - if (pVnode->status != TAOS_VN_STATUS_INIT) { + pVnode->fversion = pVnode->cversion; + vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); + if (!vnodeInInitStatus(pVnode)) { walRemoveOneOldFile(pVnode->wal); } return vnodeSaveVersion(pVnode); @@ -633,162 +459,3 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { return 0; } - -static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, - uint64_t *fversion) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while get file info", vgId); - return 0; - } - - *fversion = pVnode->fversion; - uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); - - vnodeRelease(pVnode); - return ret; -} - -static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while get wal info", vgId); - return -1; - } - - int32_t code = walGetWalFile(pVnode->wal, fileName, fileId); - - vnodeRelease(pVnode); - return code; -} - -static void vnodeNotifyRole(int32_t vgId, int8_t role) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vTrace("vgId:%d, vnode not found while notify role", vgId); - return; - } - - vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); - pVnode->role = role; - dnodeSendStatusMsgToMnode(); - - if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { - cqStart(pVnode->cq); - } else { - cqStop(pVnode->cq); - } - - vnodeRelease(pVnode); -} - -static void vnodeCtrlFlow(int32_t vgId, int32_t level) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vTrace("vgId:%d, vnode not found while flow ctrl", vgId); - return; - } - - if (pVnode->flowctrlLevel != level) { - vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); - pVnode->flowctrlLevel = level; - } - - vnodeRelease(pVnode); -} - -static int32_t vnodeResetTsdb(SVnodeObj *pVnode) { - char rootDir[128] = "\0"; - sprintf(rootDir, "%s/tsdb", pVnode->rootDir); - - if (pVnode->status != TAOS_VN_STATUS_CLOSING && pVnode->status != TAOS_VN_STATUS_INIT) { - pVnode->status = TAOS_VN_STATUS_RESET; - } - - void *tsdb = pVnode->tsdb; - pVnode->tsdb = NULL; - - // acquire vnode - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - - if (refCount > 3) { - tsem_wait(&pVnode->sem); - } - - // close tsdb, then open tsdb - tsdbCloseRepo(tsdb, 0); - STsdbAppH appH = {0}; - appH.appH = (void *)pVnode; - appH.notifyStatus = vnodeProcessTsdbStatus; - appH.cqH = pVnode->cq; - appH.cqCreateFunc = cqCreate; - appH.cqDropFunc = cqDrop; - pVnode->tsdb = tsdbOpenRepo(rootDir, &appH); - - pVnode->status = TAOS_VN_STATUS_READY; - vnodeRelease(pVnode); - - return 0; -} - -static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while notify file synced", vgId); - return 0; - } - - pVnode->fversion = fversion; - pVnode->version = fversion; - vnodeSaveVersion(pVnode); - - vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); - int32_t code = vnodeResetTsdb(pVnode); - - vnodeRelease(pVnode); - return code; -} - -void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) { - void *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while confirm forward", vgId); - return; - } - - dnodeSendRpcVWriteRsp(pVnode, wparam, code); - vnodeRelease(pVnode); -} - -static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while write to cache", vgId); - return TSDB_CODE_VND_INVALID_VGROUP_ID; - } - - int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam); - - vnodeRelease(pVnode); - return code; -} - -static int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while write to cache", vgId); - return -1; - } - - int32_t code = 0; - if (pVnode->isCommiting) { - vDebug("vgId:%d, vnode is commiting while get version", vgId); - code = -1; - } else { - *fver = pVnode->fversion; - *wver = pVnode->version; - } - - vnodeRelease(pVnode); - return code; -} diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c new file mode 100644 index 0000000000000000000000000000000000000000..8469ab12c1ef5833e73058c167afb2777ac76ba9 --- /dev/null +++ b/src/vnode/src/vnodeMgmt.c @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "dnode.h" +#include "vnodeStatus.h" +#include "vnodeWorker.h" +#include "vnodeRead.h" +#include "vnodeWrite.h" +#include "vnodeMain.h" + +static SHashObj *tsVnodesHash = NULL; + +static int32_t vnodeInitHash(void); +static void vnodeCleanupHash(void); +static void vnodeIncRef(void *ptNode); + +static SStep tsVnodeSteps[] = { + {"vnode-worker", vnodeInitMWorker, vnodeCleanupMWorker}, + {"vnode-write", vnodeInitWrite, vnodeCleanupWrite}, + {"vnode-read", vnodeInitRead, vnodeCleanupRead}, + {"vnode-hash", vnodeInitHash, vnodeCleanupHash}, + {"tsdb-queue", tsdbInitCommitQueue, tsdbDestroyCommitQueue} +}; + +int32_t vnodeInitMgmt() { + int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep); + return dnodeStepInit(tsVnodeSteps, stepSize); +} + +void vnodeCleanupMgmt() { + int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep); + dnodeStepCleanup(tsVnodeSteps, stepSize); +} + +static int32_t vnodeInitHash() { + tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (tsVnodesHash == NULL) { + vError("failed to init vnode mgmt"); + return -1; + } + + return 0; +} + +static void vnodeCleanupHash() { + if (tsVnodesHash != NULL) { + vDebug("vnode mgmt is cleanup"); + taosHashCleanup(tsVnodesHash); + tsVnodesHash = NULL; + } +} + +void *vnodeGetWal(void *pVnode) { + return ((SVnodeObj *)pVnode)->wal; +} + +void vnodeAddIntoHash(SVnodeObj *pVnode) { + taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); +} + +void vnodeRemoveFromHash(SVnodeObj *pVnode) { + taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t)); +} + +static void vnodeIncRef(void *ptNode) { + assert(ptNode != NULL); + + SVnodeObj **ppVnode = (SVnodeObj **)ptNode; + assert(ppVnode); + assert(*ppVnode); + + SVnodeObj *pVnode = *ppVnode; + atomic_add_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); +} + +void *vnodeAcquire(int32_t vgId) { + SVnodeObj **ppVnode = NULL; + if (tsVnodesHash != NULL) { + ppVnode = taosHashGetClone(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + } + + if (ppVnode == NULL || *ppVnode == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + vDebug("vgId:%d, not exist", vgId); + return NULL; + } + + return *ppVnode; +} + +void vnodeRelease(void *vparam) { + SVnodeObj *pVnode = vparam; + if (vparam == NULL) return; + + int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); + assert(refCount >= 0); + + if (refCount > 0) { + if (vnodeInResetStatus(pVnode) && refCount <= 3) { + tsem_post(&pVnode->sem); + } + } else { + vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); + vnodeDestroyInMWorker(pVnode); + int32_t count = taosHashGetSize(tsVnodesHash); + vDebug("vgId:%d, vnode is destroyed, vnodes:%d", pVnode->vgId, count); + } +} + +static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { + int64_t totalStorage = 0; + int64_t compStorage = 0; + int64_t pointsWritten = 0; + + if (!vnodeInReadyStatus(pVnode)) return; + if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; + + if (pVnode->tsdb) { + tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); + } + + SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; + pLoad->vgId = htonl(pVnode->vgId); + pLoad->dbCfgVersion = htonl(pVnode->dbCfgVersion); + pLoad->vgCfgVersion = htonl(pVnode->vgCfgVersion); + pLoad->totalStorage = htobe64(totalStorage); + pLoad->compStorage = htobe64(compStorage); + pLoad->pointsWritten = htobe64(pointsWritten); + pLoad->vnodeVersion = htobe64(pVnode->version); + pLoad->status = pVnode->status; + pLoad->role = pVnode->role; + pLoad->replica = pVnode->syncCfg.replica; +} + +int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { + + (*numOfVnodes)++; + if (*numOfVnodes >= TSDB_MAX_VNODES) { + vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES); + continue; + } else { + vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId; + } + + } + + pIter = taosHashIterate(tsVnodesHash, pIter); + } + return TSDB_CODE_SUCCESS; +} + +void vnodeBuildStatusMsg(void *param) { + SStatusMsg *pStatus = param; + + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { + vnodeBuildVloadMsg(*pVnode, pStatus); + } + pIter = taosHashIterate(tsVnodesHash, pIter); + } +} + +void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { + for (int32_t i = 0; i < numOfVnodes; ++i) { + pAccess[i].vgId = htonl(pAccess[i].vgId); + SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId); + if (pVnode != NULL) { + pVnode->accessState = pAccess[i].accessState; + if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) { + vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState); + } + vnodeRelease(pVnode); + } + } +} \ No newline at end of file diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index ed6d29505f04df25f48ffe1692d2914816001b75..acb1fc78f6554dc7cd4b984107a7bc1b95315b67 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -14,28 +14,27 @@ */ #define _DEFAULT_SOURCE -#define _NON_BLOCKING_RETRIEVE 0 #include "os.h" -#include "tglobal.h" -#include "taoserror.h" #include "taosmsg.h" -#include "query.h" -#include "trpc.h" -#include "tsdb.h" -#include "vnode.h" -#include "vnodeInt.h" #include "tqueue.h" +#include "tglobal.h" +#include "query.h" +#include "vnodeStatus.h" static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead); static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead); static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead); + static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId); -void vnodeInitReadFp(void) { +int32_t vnodeInitRead(void) { vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg; vnodeProcessReadMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg; + return 0; } +void vnodeCleanupRead() {} + // // After the fetch request enters the vnode queue, if the vnode cannot provide services, the process function are // still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the @@ -54,7 +53,7 @@ int32_t vnodeProcessRead(void *vparam, SVReadMsg *pRead) { } static int32_t vnodeCheckRead(SVnodeObj *pVnode) { - if (pVnode->status != TAOS_VN_STATUS_READY) { + if (!vnodeInReadyStatus(pVnode)) { vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], pVnode->refCount, pVnode); return TSDB_CODE_APP_NOT_READY; @@ -66,13 +65,17 @@ static int32_t vnodeCheckRead(SVnodeObj *pVnode) { return TSDB_CODE_APP_NOT_READY; } - if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica, - syncRole[pVnode->role], pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; + if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { + return TSDB_CODE_SUCCESS; + } + + if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) { + return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica, + syncRole[pVnode->role], pVnode->refCount, pVnode); + return TSDB_CODE_APP_NOT_READY; } void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) { @@ -85,22 +88,15 @@ void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) { vnodeRelease(pVnode); } -int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { - SVnodeObj *pVnode = vparam; - - if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) { - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - } - +static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, int8_t qtype, SRpcMsg *pRpcMsg) { int32_t size = sizeof(SVReadMsg) + contLen; SVReadMsg *pRead = taosAllocateQitem(size); if (pRead == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return NULL; } - if (rparam != NULL) { - SRpcMsg *pRpcMsg = rparam; + if (pRpcMsg != NULL) { pRead->rpcHandle = pRpcMsg->handle; pRead->rpcAhandle = pRpcMsg->ahandle; pRead->msgType = pRpcMsg->msgType; @@ -115,13 +111,38 @@ int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qt } pRead->qtype = qtype; - atomic_add_fetch_32(&pVnode->refCount, 1); + + return pRead; +} + +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); + if (pRead == NULL) { + assert(terrno != 0); + return terrno; + } + + SVnodeObj *pVnode = vparam; + + int32_t code = vnodeCheckRead(pVnode); + if (code != TSDB_CODE_SUCCESS) { + taosFreeQitem(pRead); + vnodeRelease(pVnode); + return code; + } + atomic_add_fetch_32(&pVnode->queuedRMsg, 1); - vTrace("vgId:%d, write into vrqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedRMsg); - taosWriteQitem(pVnode->rqueue, qtype, pRead); - return TSDB_CODE_SUCCESS; + if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRead->msgType == TSDB_MSG_TYPE_FETCH) { + vTrace("vgId:%d, write into vfetch queue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, + pVnode->queuedRMsg); + return taosWriteQitem(pVnode->fqueue, qtype, pRead); + } else { + vTrace("vgId:%d, write into vquery queue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, + pVnode->queuedRMsg); + return taosWriteQitem(pVnode->qqueue, qtype, pRead); + } } static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { @@ -131,7 +152,7 @@ static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); if (code == TSDB_CODE_SUCCESS) { - vDebug("QInfo:%p add to vread queue for exec query", *qhandle); + vTrace("QInfo:%p add to vread queue for exec query", *qhandle); } return code; @@ -162,7 +183,7 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, } } else { *freeHandle = true; - vDebug("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle); + vTrace("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle); } } else { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -197,26 +218,29 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // qHandle needs to be freed correctly if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pRead->pCont; - killQueryMsg->free = htons(killQueryMsg->free); - killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle); - - vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pRead->rpcHandle); - assert(pRead->contLen > 0 && killQueryMsg->free == 1); - - void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle); - if (qhandle == NULL || *qhandle == NULL) { - vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle, - pRead->rpcHandle); - } else { - assert(*qhandle == (void *)killQueryMsg->qhandle); - - qKillQuery(*qhandle); - qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true); - } - - return TSDB_CODE_TSC_QUERY_CANCELLED; + vError("error rpc msg in query, %s", tstrerror(pRead->code)); } +// assert(pRead->code != TSDB_CODE_RPC_NETWORK_UNAVAIL); +// if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { +// SCancelQueryMsg *pMsg = (SCancelQueryMsg *)pRead->pCont; +//// pMsg->free = htons(killQueryMsg->free); +// pMsg->qhandle = htobe64(pMsg->qhandle); +// +// vWarn("QInfo:%p connection %p broken, kill query", (void *)pMsg->qhandle, pRead->rpcHandle); +//// assert(pRead->contLen > 0 && pMsg->free == 1); +// +// void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)pMsg->qhandle); +// if (qhandle == NULL || *qhandle == NULL) { +// vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)pMsg->qhandle, pRead->rpcHandle); +// } else { +// assert(*qhandle == (void *)pMsg->qhandle); +// +// qKillQuery(*qhandle); +// qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true); +// } +// +// return TSDB_CODE_TSC_QUERY_CANCELLED; +// } int32_t code = TSDB_CODE_SUCCESS; void ** handle = NULL; @@ -261,7 +285,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } if (handle != NULL) { - vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); + vTrace("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; @@ -273,10 +297,10 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { assert(pCont != NULL); void **qhandle = (void **)pRead->qhandle; - vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); + vTrace("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); // In the retrieve blocking model, only 50% CPU will be used in query processing - if (tsHalfCoresForQuery) { + if (tsRetrieveBlockingModel) { qTableQuery(*qhandle); // do execute query qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); } else { @@ -289,7 +313,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); assert(pRead->rpcHandle != NULL); - vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, + vTrace("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, pRead->rpcHandle); // set the real rsp error code @@ -298,8 +322,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; } else { - void *h1 = qGetResultRetrieveMsg(*qhandle); - assert(h1 == NULL); + //void *h1 = qGetResultRetrieveMsg(*qhandle); + + /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */ + //assert(h1 == NULL); + freehandle = qQueryCompleted(*qhandle); } @@ -322,7 +349,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { pRetrieve->free = htons(pRetrieve->free); pRetrieve->qhandle = htobe64(pRetrieve->qhandle); - vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, + vTrace("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, pRetrieve->free, pRead->rpcHandle); memset(pRet, 0, sizeof(SRspRet)); @@ -338,11 +365,12 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } if (code != TSDB_CODE_SUCCESS) { - vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle); + vError("vgId:%d, invalid handle in retrieving result, code:%s, QInfo:%p", pVnode->vgId, tstrerror(code), (void *)pRetrieve->qhandle); vnodeBuildNoResultQueryRsp(pRet); return code; } - + + // kill current query and free corresponding resources. if (pRetrieve->free == 1) { vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); qKillQuery(*handle); @@ -373,10 +401,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); freeHandle = true; } else { // result is not ready, return immediately - assert(buildRes == true); - // Only effects in the non-blocking model - if (!tsHalfCoresForQuery) { + if (!tsRetrieveBlockingModel) { if (!buildRes) { assert(pRead->rpcHandle != NULL); @@ -401,12 +427,18 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // notify connection(handle) that current qhandle is created, if current connection from // client is broken, the query needs to be killed immediately. int32_t vnodeNotifyCurrentQhandle(void *handle, void *qhandle, int32_t vgId) { - SRetrieveTableMsg *killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); - killQueryMsg->qhandle = htobe64((uint64_t)qhandle); - killQueryMsg->free = htons(1); - killQueryMsg->header.vgId = htonl(vgId); - killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - - vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle); - return rpcReportProgress(handle, (char *)killQueryMsg, sizeof(SRetrieveTableMsg)); + SRetrieveTableMsg *pMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); + pMsg->qhandle = htobe64((uint64_t)qhandle); + pMsg->header.vgId = htonl(vgId); + pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + + vTrace("QInfo:%p register qhandle to connect:%p", qhandle, handle); + return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg)); } + +void vnodeWaitReadCompleted(SVnodeObj *pVnode) { + while (pVnode->queuedRMsg > 0) { + vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg); + taosMsleep(10); + } +} \ No newline at end of file diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c new file mode 100644 index 0000000000000000000000000000000000000000..68898435303daa8e0ac60a84cd54f4794515103e --- /dev/null +++ b/src/vnode/src/vnodeStatus.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "query.h" +#include "vnodeStatus.h" +#include "vnodeRead.h" +#include "vnodeWrite.h" + +char* vnodeStatus[] = { + "init", + "ready", + "closing", + "updating", + "reset" +}; + +bool vnodeSetInitStatus(SVnodeObj* pVnode) { + pthread_mutex_lock(&pVnode->statusMutex); + pVnode->status = TAOS_VN_STATUS_INIT; + pthread_mutex_unlock(&pVnode->statusMutex); + return true; +} + +bool vnodeSetReadyStatus(SVnodeObj* pVnode) { + bool set = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_INIT || pVnode->status == TAOS_VN_STATUS_READY || + pVnode->status == TAOS_VN_STATUS_UPDATING || pVnode->status == TAOS_VN_STATUS_RESET) { + pVnode->status = TAOS_VN_STATUS_READY; + set = true; + } + + qQueryMgmtReOpen(pVnode->qMgmt); + + pthread_mutex_unlock(&pVnode->statusMutex); + return set; +} + +static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { + bool set = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) { + pVnode->status = TAOS_VN_STATUS_CLOSING; + set = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return set; +} + +bool vnodeSetClosingStatus(SVnodeObj* pVnode) { + while (!vnodeSetClosingStatusImp(pVnode)) { + taosMsleep(1); + } + + // release local resources only after cutting off outside connections + qQueryMgmtNotifyClosed(pVnode->qMgmt); + vnodeWaitReadCompleted(pVnode); + vnodeWaitWriteCompleted(pVnode); + + return true; +} + +bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) { + bool set = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_READY) { + pVnode->status = TAOS_VN_STATUS_UPDATING; + set = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return set; +} + +static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) { + bool set = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) { + pVnode->status = TAOS_VN_STATUS_RESET; + set = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return set; +} + +bool vnodeSetResetStatus(SVnodeObj* pVnode) { + while (!vnodeSetResetStatusImp(pVnode)) { + taosMsleep(1); + } + + // release local resources only after cutting off outside connections + qQueryMgmtNotifyClosed(pVnode->qMgmt); + vnodeWaitReadCompleted(pVnode); + vnodeWaitWriteCompleted(pVnode); + + return true; +} + +bool vnodeInInitStatus(SVnodeObj* pVnode) { + bool in = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_INIT) { + in = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return in; +} + +bool vnodeInReadyStatus(SVnodeObj* pVnode) { + bool in = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_READY) { + in = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return in; +} + +bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode) { + bool in = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_UPDATING) { + in = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return in; +} + +bool vnodeInResetStatus(SVnodeObj* pVnode) { + bool in = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_RESET) { + in = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return in; +} diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c new file mode 100644 index 0000000000000000000000000000000000000000..627783c391d45b37830cb2b6d851fa6dd3261819 --- /dev/null +++ b/src/vnode/src/vnodeSync.c @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "query.h" +#include "dnode.h" +#include "vnodeVersion.h" +#include "vnodeMain.h" +#include "vnodeStatus.h" + +uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while get file info", vgId); + return 0; + } + + *fver = pVnode->fversion; + uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); + + vnodeRelease(pVnode); + return ret; +} + +int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while get wal info", vgId); + return -1; + } + + int32_t code = walGetWalFile(pVnode->wal, fileName, fileId); + + vnodeRelease(pVnode); + return code; +} + +void vnodeNotifyRole(int32_t vgId, int8_t role) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vTrace("vgId:%d, vnode not found while notify role", vgId); + return; + } + + vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); + pVnode->role = role; + dnodeSendStatusMsgToMnode(); + + if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { + cqStart(pVnode->cq); + } else { + cqStop(pVnode->cq); + } + + vnodeRelease(pVnode); +} + +void vnodeCtrlFlow(int32_t vgId, int32_t level) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vTrace("vgId:%d, vnode not found while flow ctrl", vgId); + return; + } + + if (pVnode->flowctrlLevel != level) { + vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); + pVnode->flowctrlLevel = level; + } + + vnodeRelease(pVnode); +} + +void vnodeStartSyncFile(int32_t vgId) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while start filesync", vgId); + return; + } + + vDebug("vgId:%d, datafile will be synced", vgId); + vnodeSetResetStatus(pVnode); + + vnodeRelease(pVnode); +} + +void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while stop filesync", vgId); + return; + } + + pVnode->fversion = fversion; + pVnode->version = fversion; + vnodeSaveVersion(pVnode); + + vDebug("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); + vnodeSetReadyStatus(pVnode); + + vnodeRelease(pVnode); +} + +void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) { + void *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while confirm forward", vgId); + return; + } + + dnodeSendRpcVWriteRsp(pVnode, wparam, code); + vnodeRelease(pVnode); +} + +int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while write to cache", vgId); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + + int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam); + + vnodeRelease(pVnode); + return code; +} + +int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while write to cache", vgId); + return -1; + } + + int32_t code = 0; + if (pVnode->isCommiting) { + vDebug("vgId:%d, vnode is commiting while get version", vgId); + code = -1; + } else { + *fver = pVnode->fversion; + *wver = pVnode->version; + } + + vnodeRelease(pVnode); + return code; +} + +void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code) { + SVnodeObj *pVnode = vparam; + syncConfirmForward(pVnode->sync, version, code); +} diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c index 8f6360b4f98eac8f394f5078ed2b025cea4192b0..d22bc17cbeda570f119a5a6e67637382f3a25bf8 100644 --- a/src/vnode/src/vnodeVersion.c +++ b/src/vnode/src/vnodeVersion.c @@ -15,11 +15,8 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" #include "cJSON.h" #include "tglobal.h" -#include "tsdb.h" -#include "vnodeInt.h" #include "vnodeVersion.h" int32_t vnodeReadVersion(SVnodeObj *pVnode) { @@ -44,7 +41,7 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) { goto PARSE_VER_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VER_ERROR; @@ -93,7 +90,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..6fb79d10feef74d88e04368c5607d39e7d70f6f2 --- /dev/null +++ b/src/vnode/src/vnodeWorker.c @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tutil.h" +#include "tqueue.h" +#include "tglobal.h" +#include "vnodeWorker.h" +#include "vnodeMain.h" + +typedef enum { + VNODE_WORKER_ACTION_CLEANUP, + VNODE_WORKER_ACTION_DESTROUY +} EVMWorkerAction; + +typedef struct { + int32_t vgId; + int32_t code; + void * rpcHandle; + SVnodeObj *pVnode; + EVMWorkerAction action; +} SVMWorkerMsg; + +typedef struct { + pthread_t thread; + int32_t workerId; +} SVMWorker; + +typedef struct { + int32_t curNum; + int32_t maxNum; + SVMWorker *worker; +} SVMWorkerPool; + +static SVMWorkerPool tsVMWorkerPool; +static taos_qset tsVMWorkerQset; +static taos_queue tsVMWorkerQueue; + +static void *vnodeMWorkerFunc(void *param); + +static int32_t vnodeStartMWorker() { + tsVMWorkerQueue = taosOpenQueue(); + if (tsVMWorkerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; + + taosAddIntoQset(tsVMWorkerQset, tsVMWorkerQueue, NULL); + + for (int32_t i = tsVMWorkerPool.curNum; i < tsVMWorkerPool.maxNum; ++i) { + SVMWorker *pWorker = tsVMWorkerPool.worker + i; + pWorker->workerId = i; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, vnodeMWorkerFunc, pWorker) != 0) { + vError("failed to create thread to process vmworker queue, reason:%s", strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + + tsVMWorkerPool.curNum = i + 1; + vDebug("vmworker:%d is launched, total:%d", pWorker->workerId, tsVMWorkerPool.maxNum); + } + + vDebug("vmworker queue:%p is allocated", tsVMWorkerQueue); + return TSDB_CODE_SUCCESS; +} + +int32_t vnodeInitMWorker() { + tsVMWorkerQset = taosOpenQset(); + + tsVMWorkerPool.maxNum = 1; + tsVMWorkerPool.curNum = 0; + tsVMWorkerPool.worker = calloc(sizeof(SVMWorker), tsVMWorkerPool.maxNum); + + if (tsVMWorkerPool.worker == NULL) return -1; + for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { + SVMWorker *pWorker = tsVMWorkerPool.worker + i; + pWorker->workerId = i; + vDebug("vmworker:%d is created", i); + } + + vDebug("vmworker is initialized, num:%d qset:%p", tsVMWorkerPool.maxNum, tsVMWorkerQset); + + return vnodeStartMWorker(); +} + +static void vnodeStopMWorker() { + vDebug("vmworker queue:%p is freed", tsVMWorkerQueue); + taosCloseQueue(tsVMWorkerQueue); + tsVMWorkerQueue = NULL; +} + +void vnodeCleanupMWorker() { + for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { + SVMWorker *pWorker = tsVMWorkerPool.worker + i; + if (taosCheckPthreadValid(pWorker->thread)) { + taosQsetThreadResume(tsVMWorkerQset); + } + vDebug("vmworker:%d is closed", i); + } + + for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { + SVMWorker *pWorker = tsVMWorkerPool.worker + i; + vDebug("vmworker:%d start to join", i); + if (taosCheckPthreadValid(pWorker->thread)) { + pthread_join(pWorker->thread, NULL); + } + vDebug("vmworker:%d join success", i); + } + + vDebug("vmworker is closed, qset:%p", tsVMWorkerQset); + + taosCloseQset(tsVMWorkerQset); + tsVMWorkerQset = NULL; + tfree(tsVMWorkerPool.worker); + + vnodeStopMWorker(); +} + +static int32_t vnodeWriteIntoMWorker(SVnodeObj *pVnode, EVMWorkerAction action, void *rpcHandle) { + SVMWorkerMsg *pMsg = taosAllocateQitem(sizeof(SVMWorkerMsg)); + if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY; + + pMsg->vgId = pVnode->vgId; + pMsg->pVnode = pVnode; + pMsg->rpcHandle = rpcHandle; + pMsg->action = action; + + int32_t code = taosWriteQitem(tsVMWorkerQueue, TAOS_QTYPE_RPC, pMsg); + if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS; + + return code; +} + +int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode) { + vTrace("vgId:%d, will cleanup in vmworker", pVnode->vgId); + return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_CLEANUP, NULL); +} + +int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode) { + vTrace("vgId:%d, will destroy in vmworker", pVnode->vgId); + return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_DESTROUY, NULL); +} + +static void vnodeFreeMWorkerMsg(SVMWorkerMsg *pMsg) { + vTrace("vgId:%d, disposed in vmworker", pMsg->vgId); + taosFreeQitem(pMsg); +} + +static void vnodeSendVMWorkerRpcRsp(SVMWorkerMsg *pMsg) { + if (pMsg->rpcHandle != NULL) { + SRpcMsg rpcRsp = {.handle = pMsg->rpcHandle, .code = pMsg->code}; + rpcSendResponse(&rpcRsp); + } + + vnodeFreeMWorkerMsg(pMsg); +} + +static void vnodeProcessMWorkerMsg(SVMWorkerMsg *pMsg) { + pMsg->code = 0; + + switch (pMsg->action) { + case VNODE_WORKER_ACTION_CLEANUP: + vnodeCleanUp(pMsg->pVnode); + break; + case VNODE_WORKER_ACTION_DESTROUY: + vnodeDestroy(pMsg->pVnode); + break; + default: + break; + } +} + +static void *vnodeMWorkerFunc(void *param) { + while (1) { + SVMWorkerMsg *pMsg = NULL; + if (taosReadQitemFromQset(tsVMWorkerQset, NULL, (void **)&pMsg, NULL) == 0) { + vDebug("qset:%p, vmworker got no message from qset, exiting", tsVMWorkerQset); + break; + } + + vTrace("vgId:%d, action:%d will be processed in vmworker queue", pMsg->vgId, pMsg->action); + vnodeProcessMWorkerMsg(pMsg); + vnodeSendVMWorkerRpcRsp(pMsg); + } + + return NULL; +} diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 268d1fb53b35f485ebefddf8467062b17d1950d0..99b7e7b62890c2801835d4a6130d49b9b224ddab 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -19,17 +19,9 @@ #include "taoserror.h" #include "tglobal.h" #include "tqueue.h" -#include "trpc.h" -#include "tsdb.h" -#include "twal.h" -#include "tsync.h" #include "ttimer.h" -#include "tdataformat.h" -#include "vnode.h" -#include "vnodeInt.h" -#include "syncInt.h" -#include "tcq.h" #include "dnode.h" +#include "vnodeStatus.h" #define MAX_QUEUED_MSG_NUM 10000 @@ -43,20 +35,27 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite); -void vnodeInitWriteFp(void) { +int32_t vnodeInitWrite(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessSubmitMsg; vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessCreateTableMsg; vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessDropTableMsg; vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessAlterTableMsg; vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessDropStableMsg; vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; + + return 0; } +void vnodeCleanupWrite() {} + int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { int32_t code = 0; SVnodeObj *pVnode = vparam; SWalHead * pHead = wparam; - SRspRet * pRspRet = rparam; + SVWriteMsg*pWrite = rparam; + + SRspRet *pRspRet = NULL; + if (pWrite != NULL) pRspRet = &pWrite->rspRet; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId, @@ -68,7 +67,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version); if (pHead->version == 0) { // from client or CQ - if (pVnode->status != TAOS_VN_STATUS_READY) { + if (!vnodeInReadyStatus(pVnode)) { vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId, taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version); return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state @@ -89,12 +88,15 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara // forward to peers, even it is WAL/FWD, it shall be called to update version in sync int32_t syncCode = 0; - syncCode = syncForwardToPeer(pVnode->sync, pHead, pRspRet, qtype); + syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype); if (syncCode < 0) return syncCode; // write into WAL code = walWrite(pVnode->wal, pHead); - if (code < 0) return code; + if (code < 0) { + vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code); + return code; + } pVnode->version = pHead->version; @@ -105,25 +107,25 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara return syncCode; } -static int32_t vnodeCheckWrite(void *vparam) { - SVnodeObj *pVnode = vparam; +static int32_t vnodeCheckWrite(SVnodeObj *pVnode) { if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { vDebug("vgId:%d, no write auth, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); return TSDB_CODE_VND_NO_WRITE_AUTH; } + if (pVnode->dbReplica != pVnode->syncCfg.replica && + pVnode->syncCfg.nodeInfo[pVnode->syncCfg.replica - 1].nodeId == dnodeGetDnodeId()) { + vDebug("vgId:%d, vnode is balancing and will be dropped, dbReplica:%d vgReplica:%d, refCount:%d pVnode:%p", + pVnode->vgId, pVnode->dbReplica, pVnode->syncCfg.replica, pVnode->refCount, pVnode); + return TSDB_CODE_VND_IS_BALANCING; + } + // tsdb may be in reset state if (pVnode->tsdb == NULL) { vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); return TSDB_CODE_APP_NOT_READY; } - if (pVnode->status == TAOS_VN_STATUS_CLOSING) { - vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], - pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; - } - if (pVnode->isFull) { vDebug("vgId:%d, vnode is full, refCount:%d", pVnode->vgId, pVnode->refCount); return TSDB_CODE_VND_IS_FULL; @@ -132,11 +134,6 @@ static int32_t vnodeCheckWrite(void *vparam) { return TSDB_CODE_SUCCESS; } -void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code) { - SVnodeObj *pVnode = vparam; - syncConfirmForward(pVnode->sync, version, code); -} - static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { int32_t code = TSDB_CODE_SUCCESS; @@ -177,7 +174,7 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet SMDDropTableMsg *pTable = pCont; int32_t code = TSDB_CODE_SUCCESS; - vDebug("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableId); + vDebug("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableFname); STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->tid)}; if (tsdbDropTable(pVnode->tsdb, tableId) < 0) code = terrno; @@ -200,13 +197,13 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet SDropSTableMsg *pTable = pCont; int32_t code = TSDB_CODE_SUCCESS; - vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId); + vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableFname); STableId stableId = {.uid = htobe64(pTable->uid), .tid = -1}; if (tsdbDropTable(pVnode->tsdb, stableId) < 0) code = terrno; - vDebug("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableId, tstrerror(code)); + vDebug("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableFname, tstrerror(code)); return code; } @@ -218,53 +215,81 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } -int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { - SVnodeObj *pVnode = vparam; - SWalHead * pHead = wparam; - int32_t code = 0; - - if (qtype == TAOS_QTYPE_RPC) { - code = vnodeCheckWrite(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - } - +static SVWriteMsg *vnodeBuildVWriteMsg(SVnodeObj *pVnode, SWalHead *pHead, int32_t qtype, SRpcMsg *pRpcMsg) { if (pHead->len > TSDB_MAX_WAL_SIZE) { vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version); - return TSDB_CODE_WAL_SIZE_LIMIT; + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + return NULL; } int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len; SVWriteMsg *pWrite = taosAllocateQitem(size); if (pWrite == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return NULL; } - if (rparam != NULL) { - SRpcMsg *pRpcMsg = rparam; + if (pRpcMsg != NULL) { pWrite->rpcMsg = *pRpcMsg; } - memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len); + memcpy(&pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len); pWrite->pVnode = pVnode; pWrite->qtype = qtype; atomic_add_fetch_32(&pVnode->refCount, 1); + return pWrite; +} + +static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) { + SVnodeObj *pVnode = pWrite->pVnode; + + if (pWrite->qtype == TAOS_QTYPE_RPC) { + int32_t code = vnodeCheckWrite(pVnode); + if (code != TSDB_CODE_SUCCESS) { + vError("vgId:%d, failed to write into vwqueue since %s", pVnode->vgId, tstrerror(code)); + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + return code; + } + } + + if (!vnodeInReadyOrUpdatingStatus(pVnode)) { + vError("vgId:%d, failed to write into vwqueue, vstatus is %s, refCount:%d pVnode:%p", pVnode->vgId, + vnodeStatus[pVnode->status], pVnode->refCount, pVnode); + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + return TSDB_CODE_APP_NOT_READY; + } + int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); if (queued > MAX_QUEUED_MSG_NUM) { - vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued); - taosMsleep(1); + int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3; + if (ms > 100) ms = 100; + vDebug("vgId:%d, too many msg:%d in vwqueue, flow control %dms", pVnode->vgId, queued, ms); + taosMsleep(ms); } - code = vnodePerformFlowCtrl(pWrite); - if (code != 0) return 0; - vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); - taosWriteQitem(pVnode->wqueue, qtype, pWrite); + taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); return TSDB_CODE_SUCCESS; } +int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { + SVWriteMsg *pWrite = vnodeBuildVWriteMsg(vparam, wparam, qtype, rparam); + if (pWrite == NULL) { + assert(terrno != 0); + return terrno; + } + + int32_t code = vnodePerformFlowCtrl(pWrite); + if (code != 0) return 0; + + return vnodeWriteToWQueueImp(pWrite); +} + void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { SVnodeObj *pVnode = vparam; @@ -278,10 +303,12 @@ void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { SVWriteMsg *pWrite = param; SVnodeObj * pVnode = pWrite->pVnode; - int32_t code = TSDB_CODE_VND_SYNCING; + int32_t code = TSDB_CODE_VND_IS_SYNCING; + + if (pVnode->flowctrlLevel <= 0) code = TSDB_CODE_VND_IS_FLOWCTRL; pWrite->processedCount++; - if (pWrite->processedCount > 100) { + if (pWrite->processedCount >= 100) { vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), pWrite->processedCount); pWrite->processedCount = 1; @@ -292,18 +319,21 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->processedCount); pWrite->processedCount = 0; - taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); + code = vnodeWriteToWQueueImp(pWrite); + if (code != 0) { + dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } } } } static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { SVnodeObj *pVnode = pWrite->pVnode; - if (pVnode->flowctrlLevel <= 0) return 0; if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; + if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; - if (tsFlowCtrl == 0) { - int32_t ms = pow(2, pVnode->flowctrlLevel + 2); + if (tsEnableFlowCtrl == 0) { + int32_t ms = (int32_t)pow(2, pVnode->flowctrlLevel + 2); if (ms > 100) ms = 100; vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); taosMsleep(ms); @@ -317,3 +347,10 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { return TSDB_CODE_VND_ACTION_IN_PROGRESS; } } + +void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { + while (pVnode->queuedWMsg > 0) { + vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); + taosMsleep(10); + } +} diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index 359e09287a9409a7acffe27d934884cce0510bb0..a89024dab5060b1f18174f769e0d70c00ad00faf 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -1,11 +1,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +ADD_DEFINITIONS(-DWAL_CHECKSUM_WHOLE) + INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) - ADD_LIBRARY(twal ${SRC}) - TARGET_LINK_LIBRARIES(twal tutil common) - ADD_SUBDIRECTORY(test) -ENDIF () +ADD_LIBRARY(twal ${SRC}) +TARGET_LINK_LIBRARIES(twal tutil common) +ADD_SUBDIRECTORY(test) diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h index 06748d885f7b0fb798abc1873a1734abbf9f05ab..890b404ce9489f0f7798aa7ff21b4f1e80da2b04 100644 --- a/src/wal/inc/walInt.h +++ b/src/wal/inc/walInt.h @@ -38,7 +38,7 @@ extern int32_t wDebugFlag; #define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE)) #define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) #define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_FILE_NUM 3 +#define WAL_FILE_NUM 1 // 3 typedef struct { uint64_t version; diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 72ea239817778db036e1eb05fc67fdcd128cbb6b..39ce2657aa2c27c2249d1492f5dbcde2517cc988 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -44,7 +44,7 @@ int32_t walInit() { return code; } - wInfo("wal module is initialized, refId:%d", tsWal.refId); + wInfo("wal module is initialized, rsetId:%d", tsWal.refId); return code; } @@ -203,14 +203,14 @@ static int32_t walCreateThread() { } pthread_attr_destroy(&thAttr); - wDebug("wal thread is launched"); + wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); return TSDB_CODE_SUCCESS; } static void walStopThread() { tsWal.stop = 1; - if (tsWal.thread) { + if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 10e1b4dd614fb881ea8819a16ca2451b20f4393d..aeb49830299eb0dcddfbd39a7a838fdc5d45b081 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -39,7 +39,7 @@ int32_t walRenew(void *handle) { if (tfValid(pWal->tfd)) { tfClose(pWal->tfd); - wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name); + wDebug("vgId:%d, file:%s, it is closed while renew", pWal->vgId, pWal->name); } if (pWal->keep == TAOS_WAL_KEEP) { @@ -56,7 +56,7 @@ int32_t walRenew(void *handle) { code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); } else { - wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name); + wDebug("vgId:%d, file:%s, it is created and open while renew", pWal->vgId, pWal->name); } pthread_mutex_unlock(&pWal->mutex); @@ -95,11 +95,15 @@ void walRemoveAllOldFiles(void *handle) { int64_t fileId = -1; pthread_mutex_lock(&pWal->mutex); + + tfClose(pWal->tfd); + wDebug("vgId:%d, file:%s, it is closed before remove all wals", pWal->vgId, pWal->name); + while (walGetNextFile(pWal, &fileId) >= 0) { snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); if (remove(pWal->name) < 0) { - wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name); + wError("vgId:%d, wal:%p file:%s, failed to remove since %s", pWal->vgId, pWal, pWal->name, strerror(errno)); } else { wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); } @@ -107,6 +111,28 @@ void walRemoveAllOldFiles(void *handle) { pthread_mutex_unlock(&pWal->mutex); } +#if defined(WAL_CHECKSUM_WHOLE) + +static void walUpdateChecksum(SWalHead *pHead) { + pHead->sver = 1; + pHead->cksum = 0; + pHead->cksum = taosCalcChecksum(0, (uint8_t *)pHead, sizeof(*pHead) + pHead->len); +} + +static int walValidateChecksum(SWalHead *pHead) { + if (pHead->sver == 0) { // for compatible with wal before sver 1 + return taosCheckChecksumWhole((uint8_t *)pHead, sizeof(*pHead)); + } else if (pHead->sver == 1) { + uint32_t cksum = pHead->cksum; + pHead->cksum = 0; + return taosCheckChecksum((uint8_t *)pHead, sizeof(*pHead) + pHead->len, cksum); + } + + return 0; +} + +#endif + int32_t walWrite(void *handle, SWalHead *pHead) { if (handle == NULL) return -1; @@ -119,7 +145,13 @@ int32_t walWrite(void *handle, SWalHead *pHead) { if (pHead->version <= pWal->version) return 0; pHead->signature = WAL_SIGNATURE; +#if defined(WAL_CHECKSUM_WHOLE) + walUpdateChecksum(pHead); +#else + pHead->sver = 0; taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead)); +#endif + int32_t contLen = pHead->len + sizeof(SWalHead); pthread_mutex_lock(&pWal->mutex); @@ -166,14 +198,14 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { char walName[WAL_FILE_LEN]; snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); - wDebug("vgId:%d, file:%s, will be restored", pWal->vgId, walName); + wInfo("vgId:%d, file:%s, will be restored", pWal->vgId, walName); int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId); if (code != TSDB_CODE_SUCCESS) { wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code)); continue; } - wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName); + wInfo("vgId:%d, file:%s, restore success, wver:%" PRIu64, pWal->vgId, walName, pWal->version); count++; } @@ -192,7 +224,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - wDebug("vgId:%d, file:%s open success", pWal->vgId, pWal->name); + wDebug("vgId:%d, file:%s, it is created and open while restore", pWal->vgId, pWal->name); } return TSDB_CODE_SUCCESS; @@ -242,16 +274,40 @@ static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, continue; } +#if defined(WAL_CHECKSUM_WHOLE) + if (pHead->sver == 0 && walValidateChecksum(pHead)) { + wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos); + *offset = pos; + return TSDB_CODE_SUCCESS; + } + + if (pHead->sver == 1) { + if (tfRead(tfd, pHead->cont, pHead->len) < pHead->len) { + wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); + return TSDB_CODE_WAL_FILE_CORRUPTED; + } + + if (walValidateChecksum(pHead)) { + wInfo("vgId:%d, wal whole cksum check passed, offset:%" PRId64, pWal->vgId, pos); + *offset = pos; + return TSDB_CODE_SUCCESS; + } + } + +#else if (taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos); *offset = pos; return TSDB_CODE_SUCCESS; } + +#endif } return TSDB_CODE_WAL_FILE_CORRUPTED; } + static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) { int32_t size = WAL_MAX_SIZE; void * buffer = tmalloc(size); @@ -265,16 +321,16 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); tfree(buffer); return TAOS_SYSTEM_ERROR(errno); + } else { + wDebug("vgId:%d, file:%s, open for restore", pWal->vgId, name); } - wDebug("vgId:%d, file:%s, start to restore", pWal->vgId, name); - int32_t code = TSDB_CODE_SUCCESS; int64_t offset = 0; SWalHead *pHead = buffer; while (1) { - int32_t ret = tfRead(tfd, pHead, sizeof(SWalHead)); + int32_t ret = (int32_t)tfRead(tfd, pHead, sizeof(SWalHead)); if (ret == 0) break; if (ret < 0) { @@ -289,7 +345,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch break; } - if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { +#if defined(WAL_CHECKSUM_WHOLE) + if (pHead->sver == 0 && !walValidateChecksum(pHead)) { wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); @@ -299,19 +356,61 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } } - if (pHead->len > size - sizeof(SWalHead)) { - size = sizeof(SWalHead) + pHead->len; - buffer = realloc(buffer, size); - if (buffer == NULL) { - wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); - code = TAOS_SYSTEM_ERROR(errno); + if (pHead->len < 0 || pHead->len > size - sizeof(SWalHead)) { + wError("vgId:%d, file:%s, wal head len out of range, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); + break; + } + } + + ret = (int32_t)tfRead(tfd, pHead->cont, pHead->len); + if (ret < 0) { + wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); + break; + } + + if (ret < pHead->len) { + wError("vgId:%d, file:%s, failed to read wal body, ret:%d len:%d", pWal->vgId, name, ret, pHead->len); + offset += sizeof(SWalHead); + continue; + } + + if (pHead->sver == 1 && !walValidateChecksum(pHead)) { + wError("vgId:%d, file:%s, wal whole cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); break; } + } - pHead = buffer; +#else + if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { + wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); + break; + } + } + + if (pHead->len < 0 || pHead->len > size - sizeof(SWalHead)) { + wError("vgId:%d, file:%s, wal head len out of range, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); + break; + } } - ret = tfRead(tfd, pHead->cont, pHead->len); + ret = (int32_t)tfRead(tfd, pHead->cont, pHead->len); if (ret < 0) { wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); code = TAOS_SYSTEM_ERROR(errno); @@ -324,9 +423,10 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch continue; } +#endif offset = offset + sizeof(SWalHead) + pHead->len; - wDebug("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, + wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, fileId, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; @@ -336,6 +436,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch tfClose(tfd); tfree(buffer); + wDebug("vgId:%d, file:%s, it is closed after restore", pWal->vgId, name); return code; } diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt index 6c232ce4b91eecf7e8818922bbdbebbfe380e444..f20a57899e049115ded0012c0092bf643af76187 100644 --- a/src/wal/test/CMakeLists.txt +++ b/src/wal/test/CMakeLists.txt @@ -10,4 +10,12 @@ IF (TD_LINUX) ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(../inc) + + LIST(APPEND WALTEST_SRC ./waltest.c) + ADD_EXECUTABLE(waltest ${WALTEST_SRC}) + TARGET_LINK_LIBRARIES(waltest twal osdetail tutil) + +ENDIF () diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 7a473ed18c958afa8be3c5b94b04d2fd548a56fd..9a52a2ca833e671fe96acf6800f15e44fba292e3 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -76,7 +76,7 @@ int main(int argc, char *argv[]) { taosInitLog("wal.log", 100000, 10); - SWalCfg walCfg; + SWalCfg walCfg = {0}; walCfg.walLevel = level; walCfg.keep = keep; diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..7cdcfb2e24f92a96e2d3cc15324ca4f3ac4bc7a8 --- /dev/null +++ b/tests/Jenkinsfile @@ -0,0 +1,291 @@ +def pre_test(){ + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + sudo rmtaos + ''' + } + sh ''' + cd ${WKC} + git reset --hard + git checkout $BRANCH_NAME + git pull + git submodule update + cd ${WK} + git reset --hard + git checkout $BRANCH_NAME + git pull + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake .. > /dev/null + make > /dev/null + make install > /dev/null + ''' + return 1 +} +pipeline { + agent none + environment{ + + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDinternal/community' + } + + stages { + stage('Parallel test stage') { + parallel { + stage('pytest') { + agent{label '184'} + steps { + pre_test() + sh ''' + cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh pytest + date''' + } + } + stage('test_b1') { + agent{label 'master'} + steps { + pre_test() + + sh ''' + cd ${WKC}/tests + ./test-all.sh b1 + date''' + sh ''' + cd ${WKC}/tests + ./test-all.sh full jdbc + date''' + sh ''' + cd ${WKC}/tests + ./test-all.sh full unit + date''' + } + } + + stage('test_crash_gen') { + agent{label "185"} + steps { + pre_test() + sh ''' + cd ${WKC}/tests/pytest + ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./crash_gen.sh -a -p -t 4 -s 2000 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./handle_crash_gen_val_log.sh + ''' + } + sh''' + systemctl start taosd + sleep 10 + ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/gotest + bash batchtest.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single -DskipTests >/dev/null + java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/src/connector/jdbc + mvn clean package -Dmaven.test.skip=true >/dev/null + cd ${WKC}/tests/examples/JDBC/JDBCDemo/ + java --class-path=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cp -rf ${WKC}/tests/examples/nodejs ${JENKINS_HOME}/workspace/ + cd ${JENKINS_HOME}/workspace/nodejs + node nodejsChecker.js host=localhost + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# + dotnet run + ''' + } + sh ''' + systemctl stop taosd + cd ${WKC}/tests + ./test-all.sh b2 + date + ''' + } + } + + stage('test_valgrind') { + agent{label "186"} + + steps { + pre_test() + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + nohup taosd >/dev/null & + sleep 10 + python3 concurrent_inquiry.py -c 1 + + ''' + } + sh ''' + cd ${WKC}/tests/pytest + ./valgrind-test.sh 2>&1 > mem-error-out.log + ./handle_val_log.sh + + date + cd ${WKC}/tests + ./test-all.sh b3 + date''' + } + } + + stage('arm64_build'){ + agent{label 'arm64'} + steps{ + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + } + stage('arm32_build'){ + agent{label 'arm32'} + steps{ + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + + } + } + } + } + + } + post { + success { + emailext ( + subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

    + 构建信息 +
    +
      +
      +
    • 构建名称>>分支：${PROJECT_NAME}
      • +
    • 构建结果： Successful
      • +
    • 构建编号：${BUILD_NUMBER}
      • +
    • 触发用户：${CAUSE}
      • +
    • 变更概要：${CHANGES}
      • +
    • 构建地址：${BUILD_URL}
      • +
    • 构建日志：${BUILD_URL}console
      • +
    • 变更集：${JELLY_SCRIPT}
      • +
      +
    +
    + + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

    + 构建信息 +
    +
      +
      +
    • 构建名称>>分支：${PROJECT_NAME}
      • +
    • 构建结果： Successful
      • +
    • 构建编号：${BUILD_NUMBER}
      • +
    • 触发用户：${CAUSE}
      • +
    • 变更概要：${CHANGES}
      • +
    • 构建地址：${BUILD_URL}
      • +
    • 构建日志：${BUILD_URL}console
      • +
    • 变更集：${JELLY_SCRIPT}
      • +
      +
    +
    + + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + } +} \ No newline at end of file diff --git a/tests/comparisonTest/tdengine/CMakeLists.txt b/tests/comparisonTest/tdengine/CMakeLists.txt index 990612b8c37a47fb9f4dc29e25f3e5c6840ff199..36ed3efe191c9d949d6234bd61ffbbe28c3a33d2 100644 --- a/tests/comparisonTest/tdengine/CMakeLists.txt +++ b/tests/comparisonTest/tdengine/CMakeLists.txt @@ -5,3 +5,8 @@ IF (TD_LINUX) add_executable(tdengineTest tdengineTest.c) target_link_libraries(tdengineTest taos_static tutil common pthread) ENDIF() + +IF (TD_DARWIN) + add_executable(tdengineTest tdengineTest.c) + target_link_libraries(tdengineTest taos_static tutil common pthread) +ENDIF() diff --git a/tests/examples/C#/taosdemo/README.md b/tests/examples/C#/taosdemo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2d125fb140076c46c9abc4c60db330b28b494802 --- /dev/null +++ b/tests/examples/C#/taosdemo/README.md @@ -0,0 +1,36 @@ +install build environment +=== +yum/apt install mono-complete + +build C# version taosdemo +=== +mcs -out:taosdemo *.cs + +run C# version taosdemo +=== +Usage: mono taosdemo.exe [OPTION...] + + --help Show usage. + + -h host, The host to connect to TDengine. Default is localhost. + -p port, The TCP/IP port number to use for the connection. Default is 0. + -u user, The user name to use when connecting to the server. Default is 'root'. + -P password, The password to use when connecting to the server. Default is 'taosdata'. + -d database, Destination database. Default is 'test'. + -a replica, Set the replica parameters of the database, Default 1, min: 1, max: 5. + -m table_prefix, Table prefix name. Default is 't'. + -M stable, Use super table. + -s stable_prefix, STable prefix name. Default is 'st' + -Q query, Execute query command. set 'DEFAULT' means select * from each table + -T num_of_threads, The number of threads. Default is 10. + -r num_of_records_per_req, The number of records per request. Default is 1000. + -t num_of_tables, The number of tables. Default is 1. + -n num_of_records_per_table, The number of records per table. Default is 1. + -c config_directory, Configuration directory. Default is '/etc/taos/'. + -x flag, Insert only flag. + -O order, Insert mode--0: In order, 1: Out of order. Default is in order. + -R rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50. + -D Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database. + -v Print verbose output + -g Print debug output + -y Skip read key for continous test, default is not skip diff --git a/tests/examples/C#/taosdemo/TDengineDriver.cs b/tests/examples/C#/taosdemo/TDengineDriver.cs new file mode 100644 index 0000000000000000000000000000000000000000..205269501d376a4753b3aedbfa8d512b2df31600 --- /dev/null +++ b/tests/examples/C#/taosdemo/TDengineDriver.cs @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace TDengineDriver +{ + enum TDengineDataType + { + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10 // unicode string + } + + enum TDengineInitOption + { + TSDB_OPTION_LOCALE = 0, + TSDB_OPTION_CHARSET = 1, + TSDB_OPTION_TIMEZONE = 2, + TDDB_OPTION_CONFIGDIR = 3, + TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 + } + + class TDengineMeta + { + public string name; + public short size; + public byte type; + public string TypeName() + { + switch ((TDengineDataType)type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + return "BOOLEAN"; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + return "BYTE"; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + return "SHORT"; + case TDengineDataType.TSDB_DATA_TYPE_INT: + return "INT"; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + return "LONG"; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + return "STRING"; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + return "NCHAR"; + default: + return "undefine"; + } + } + } + + class TDengine + { + public const int TSDB_CODE_SUCCESS = 0; + + [DllImport("taos", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] + static extern public void Init(); + + [DllImport("taos", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); + + [DllImport("taos", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] + static extern public void Options(int option, string value); + + [DllImport("taos", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); + + [DllImport("taos", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) + { + IntPtr errPtr = taos_errstr(res); + return Marshal.PtrToStringAnsi(errPtr); + } + + [DllImport("taos", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); + + [DllImport("taos", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) + { + const int fieldSize = 68; + + List metas = new List(); + if (res == IntPtr.Zero) + { + return metas; + } + + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); + + for (int i = 0; i < fieldCount; ++i) + { + int offset = i * fieldSize; + + TDengineMeta meta = new TDengineMeta(); + meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); + metas.Add(meta); + } + + return metas; + } + + [DllImport("taos", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); + + [DllImport("taos", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); + } +} diff --git a/tests/examples/C#/taosdemo/taosdemo.cs b/tests/examples/C#/taosdemo/taosdemo.cs new file mode 100644 index 0000000000000000000000000000000000000000..2d78418e0ab8599f87a544b833b9b6bbd0b48ee7 --- /dev/null +++ b/tests/examples/C#/taosdemo/taosdemo.cs @@ -0,0 +1,858 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +using System; +using System.Text; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Collections; +using System.Threading; +using System.Diagnostics; + +namespace TDengineDriver +{ + class TDengineTest + { + //connect parameters + private string host = "127.0.0.1"; + private string configDir = "C:/TDengine/cfg"; + private string user = "root"; + private string password = "taosdata"; + private short port = 0; + + //sql parameters + private string dbName = "db"; + private string stablePrefix = "st"; + private string tablePrefix = "t"; + + private bool isInsertOnly = false; + private string query = "NONE"; + private short queryMode = 1; + + private long recordsPerTable = 1; + private int recordsPerRequest = 1; + private int colsPerRecord = 3; + private long batchRows = 1000; + private long numOfTables = 1; + private short replica = 1; + + private IntPtr conn = IntPtr.Zero; + // private long rowsInserted = 0; + private bool useStable = false; + private short methodOfDelete = 0; + private long numOfThreads = 1; + private short rateOfOutorder = 10; + private bool order = true; + private bool skipReadKey = false; + private bool verbose = false; + private bool debug = false; + + + static void HelpPrint(string arg, string desc) + { + string indent = " "; + Console.WriteLine("{0}{1}", indent, arg.PadRight(25)+desc); + } + + static void PrintHelp(String[] argv) + { + for (int i = 0; i < argv.Length; ++i) + { + if ("--help" == argv[i]) + { + Console.WriteLine("Usage: mono taosdemo.exe [OPTION...]"); + Console.WriteLine(""); + HelpPrint("--help", "Show usage."); + Console.WriteLine(""); + + HelpPrint("-h ", "host, The host to connect to TDengine. Default is localhost."); + HelpPrint("-p ", "port, The TCP/IP port number to use for the connection. Default is 0."); + HelpPrint("-u ", "user, The user name to use when connecting to the server. Default is 'root'."); + HelpPrint("-P ", "password, The password to use when connecting to the server. Default is 'taosdata'."); + HelpPrint("-d ", "database, Destination database. Default is 'test'."); + HelpPrint("-a ", "replica, Set the replica parameters of the database, Default 1, min: 1, max: 5."); + HelpPrint("-m
    ", "table_prefix, Table prefix name. Default is 't'."); + HelpPrint("-M", "stable, Use super table."); + HelpPrint("-s ", "stable_prefix, STable prefix name. Default is 'st'"); + HelpPrint("-Q ", "query, Execute query command. set 'DEFAULT' means select * from each table"); + /* NOT SUPPORT SO FAR + HelpPrint("-o", "outputfile, Direct output to the named file. Default is './output.txt'."); + HelpPrint("-q", "query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + HelpPrint("-b", "type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'."); + HelpPrint("-w", "length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8"); + HelpPrint("-l", "num_of_cols_per_record, The number of columns per record. Default is 3."); + */ + HelpPrint("-T ", "num_of_threads, The number of threads. Default is 10."); + HelpPrint("-r ", "num_of_records_per_req, The number of records per request. Default is 1000."); + HelpPrint("-t ", "num_of_tables, The number of tables. Default is 1."); + HelpPrint("-n ", "num_of_records_per_table, The number of records per table. Default is 1."); + HelpPrint("-c ", "config_directory, Configuration directory. Default is '/etc/taos/'."); + HelpPrint("-x", "flag, Insert only flag."); + HelpPrint("-O", "order, Insert mode--0: In order, 1: Out of order. Default is in order."); + HelpPrint("-R ", "rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50."); + HelpPrint("-D ", "Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database."); + HelpPrint("-v", "Print verbose output"); + HelpPrint("-g", "Print debug output"); + HelpPrint("-y", "Skip read key for continous test, default is not skip"); + + System.Environment.Exit(0); + } + } + } + + public void ReadArgument(String[] argv) + { + host = this.GetArgumentAsString(argv, "-h", "127.0.0.1"); + port = (short)this.GetArgumentAsLong(argv, "-p", 0, 65535, 6030); + user = this.GetArgumentAsString(argv, "-u", "root"); + password = this.GetArgumentAsString(argv, "-P", "taosdata"); + dbName = this.GetArgumentAsString(argv, "-d", "db"); + stablePrefix = this.GetArgumentAsString(argv, "-s", "st"); + tablePrefix = this.GetArgumentAsString(argv, "-m", "t"); + isInsertOnly = this.GetArgumentAsFlag(argv, "-x", true); + query = this.GetArgumentAsString(argv, "-Q", "NONE"); + queryMode = (short)this.GetArgumentAsLong(argv, "-q", 0, 1, 0); + numOfTables = this.GetArgumentAsLong(argv, "-t", 1, 1000000000, 1); + batchRows = this.GetArgumentAsLong(argv, "-r", 1, 10000, 1000); + recordsPerTable = this.GetArgumentAsLong(argv, "-n", 1, 100000000000, 1); + recordsPerRequest = (int)this.GetArgumentAsLong(argv, "-r", 1, 10000, 1); + colsPerRecord = (int)this.GetArgumentAsLong(argv, "-l", 1, 1024, 3); + configDir = this.GetArgumentAsString(argv, "-c", "C:/TDengine/cfg"); + useStable = this.GetArgumentAsFlag(argv, "-M", true); + + replica = (short)this.GetArgumentAsLong(argv, "-a", 1, 5, 1); + methodOfDelete = (short)this.GetArgumentAsLong(argv, "-D", 0, 3, 0); + numOfThreads = (short)this.GetArgumentAsLong(argv, "-T", 1, 10000, 1); + order = this.GetArgumentAsFlag(argv, "-O", false); + rateOfOutorder = (short)this.GetArgumentAsLong(argv, "-R", 0, 50, 10); + + skipReadKey = this.GetArgumentAsFlag(argv, "-y", true); + verbose = this.GetArgumentAsFlag(argv, "-v", true); + debug = this.GetArgumentAsFlag(argv, "-g", true); + + VerbosePrint ("###################################################################\n"); + VerbosePrintFormat ("# Server IP: {0}\n", host); + VerbosePrintFormat ("# User: {0}\n", user); + VerbosePrintFormat ("# Password: {0}\n", password); + VerbosePrintFormat ("# Number of Columns per record: {0}\n", colsPerRecord); + VerbosePrintFormat ("# Number of Threads: {0}\n", numOfThreads); + VerbosePrintFormat ("# Number of Tables: {0}\n", numOfTables); + VerbosePrintFormat ("# Number of records per Table: {0}\n", recordsPerTable); + VerbosePrintFormat ("# Records/Request: {0}\n", recordsPerRequest); + VerbosePrintFormat ("# Database name: {0}\n", dbName); + VerbosePrintFormat ("# Replica: {0}\n", replica); + VerbosePrintFormat ("# Use STable: {0}\n", useStable); + VerbosePrintFormat ("# Table prefix: {0}\n", tablePrefix); + if (useStable == true) + { + VerbosePrintFormat("# STable prefix: {0}\n", stablePrefix); + } + VerbosePrintFormat ("# Data order: {0}\n", order); + VerbosePrintFormat ("# Data out of order rate: {0}\n", rateOfOutorder); + VerbosePrintFormat ("# Delete method: {0}\n", methodOfDelete); + VerbosePrintFormat ("# Query command: {0}\n", query); + VerbosePrintFormat ("# Query Mode: {0}\n", queryMode); + VerbosePrintFormat ("# Insert Only: {0}\n", isInsertOnly); + VerbosePrintFormat ("# Verbose output {0}\n", verbose); + VerbosePrintFormat ("# Test time: {0}\n", DateTime.Now.ToString("h:mm:ss tt")); + + VerbosePrint ("###################################################################\n"); + + if (skipReadKey == false) + { + Console.Write("Press any key to continue..\n"); + Console.ReadKey(); + } + } + + public bool GetArgumentAsFlag(String[] argv, String argName, bool defaultValue) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName == argv[i]) + { + return defaultValue; + } + } + return !defaultValue; + } + + public long GetArgumentAsLong(String[] argv, String argName, int minVal, long maxVal, int defaultValue) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName != argv[i]) + { + continue; + } + if (i < argc - 1) + { + String tmp = argv[i + 1]; + if (tmp[0] == '-') + { + Console.WriteLine("option {0:G} requires an argument", argName); + ExitProgram(1); + } + + long tmpVal = Convert.ToInt64(tmp); + if (tmpVal < minVal || tmpVal > maxVal) + { + Console.WriteLine("option {0:G} value should in range [{1:G}, {2:G}]", argName, minVal, maxVal); + ExitProgram(1); + } + + return tmpVal; + } + } + + return defaultValue; + } + + public String GetArgumentAsString(String[] argv, String argName, String defaultValue) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName != argv[i]) + { + continue; + } + if (i < argc - 1) + { + String tmp = argv[i + 1]; + if (tmp[0] == '-') + { + Console.WriteLine("option {0:G} requires an argument", argName); + ExitProgram(1); + } + return tmp; + } + } + + return defaultValue; + } + + static void CleanAndExitProgram(int ret) + { + TDengine.Cleanup(); + System.Environment.Exit(ret); + } + + static void ExitProgram(int ret) + { + System.Environment.Exit(ret); + } + + private void VerbosePrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void VerbosePrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (debug == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (debug == true) + { + Console.Write(str); + } + } + + public void InitTDengine() + { + TDengine.Options((int)TDengineInitOption.TDDB_OPTION_CONFIGDIR, this.configDir); + TDengine.Options((int)TDengineInitOption.TDDB_OPTION_SHELL_ACTIVITY_TIMER, "60"); + TDengine.Init(); + VerbosePrint("TDengine Initialization finished\n"); + } + + public void ConnectTDengine() + { + string db = ""; + VerbosePrintFormat("host:{0} user:{1}, pass:{2}; db:{3}, port:{4}\n", + this.host, this.user, this.password, db, this.port); + this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); + if (this.conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + CleanAndExitProgram(1); + } + else + { + VerbosePrint("Connect to TDengine success\n"); + } + } + + public void CreateTablesByThreads() + { + Thread[] threadArr = new Thread[numOfThreads]; + + long quotition = numOfTables / numOfThreads; + if (quotition < 1) + { + numOfThreads = numOfTables; + quotition = 1; + } + + long remainder = 0; + if (numOfThreads != 0) + { + remainder = numOfTables % numOfThreads; + } + + long last = 0; + + for (int i = 0; i < numOfThreads; i++) + { + CreateTableThread createTableThread = new CreateTableThread(); + createTableThread.id = i; + createTableThread.verbose = verbose; + createTableThread.debug = debug; + createTableThread.dbName = this.dbName; + createTableThread.tablePrefix = this.tablePrefix; + createTableThread.useStable = useStable; + if (useStable) + { + createTableThread.stablePrefix = stablePrefix; + } + createTableThread.conn = conn; + + createTableThread.start = last; + if (i < remainder) + { + createTableThread.end = last + quotition; + } + else + { + createTableThread.end = last + quotition - 1; + } + last = createTableThread.end + 1; + + threadArr[i] = new Thread(createTableThread.ThreadMain); + threadArr[i].Start(); + threadArr[i].Join(); + } + } + + public void dropDatabase() + { + StringBuilder sql = new StringBuilder(); + sql.Append("DROP DATABASE IF EXISTS ").Append(this.dbName); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + VerbosePrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + CleanAndExitProgram(1); + } + + } + + public void CreateDb() + { + StringBuilder sql = new StringBuilder(); + sql.Append("CREATE DATABASE IF NOT EXISTS ").Append(this.dbName).Append(" replica ").Append(this.replica); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + VerbosePrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + CleanAndExitProgram(1); + } + TDengine.FreeResult(res); + } + + public void CreateStable() + { + StringBuilder sql = new StringBuilder(); + + sql.Clear(); + sql.Append("CREATE TABLE IF NOT EXISTS "). + Append(this.dbName).Append(".").Append(this.stablePrefix). + Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + VerbosePrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + CleanAndExitProgram(1); + } + TDengine.FreeResult(res); + } + + public void InsertByThreads() + { + Thread[] threadArr = new Thread[numOfThreads]; + + long quotition = numOfTables / numOfThreads; + if (quotition < 1) + { + numOfThreads = numOfTables; + quotition = 1; + } + + long remainder = 0; + if (numOfThreads != 0) + { + remainder = numOfTables % numOfThreads; + } + + long last = 0; + + for (int i = 0; i < numOfThreads; i++) + { + InsertDataThread insertThread = new InsertDataThread(); + insertThread.id = i; + insertThread.recordsPerTable = recordsPerTable; + insertThread.batchRows = batchRows; + insertThread.numOfTables = numOfTables; + insertThread.verbose = verbose; + insertThread.debug = debug; + insertThread.dbName = this.dbName; + insertThread.tablePrefix = this.tablePrefix; + insertThread.order = this.order; + insertThread.rateOfOutorder = this.rateOfOutorder; + if (useStable) + { + insertThread.stablePrefix = stablePrefix; + } + insertThread.conn = conn; + + insertThread.start = last; + if (i < remainder) + { + insertThread.end = last + quotition; + } + else + { + insertThread.end = last + quotition - 1; + } + last = insertThread.end + 1; + + threadArr[i] = new Thread(insertThread.ThreadMain); + threadArr[i].Start(); + threadArr[i].Join(); + } + } + + public void ExecuteQuery() + { + long queryRows = 0; + + for (int i = 0; i < this.numOfTables; ++i) + { + string sql; + + if (query == "DEFAULT") + { + sql = "select * from " + this.dbName + "." + tablePrefix + i; + } + else + { + sql = query; + } + DebugPrintFormat("query: {0}, sql:{1}\n", query, sql); + + IntPtr res = TDengine.Query(conn, sql); + DebugPrintFormat("res: {0}\n", res); + if (res == IntPtr.Zero) + { + Console.WriteLine(sql + " failure, reason: " + TDengine.Error(res)); + CleanAndExitProgram(1); + } + + int fieldCount = TDengine.FieldCount(res); + DebugPrint("field count: " + fieldCount + "\n"); + + List metas = TDengine.FetchFields(res); + for (int j = 0; j < metas.Count; j++) + { + TDengineMeta meta = (TDengineMeta)metas[j]; + DebugPrint("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size + "\n"); + } + + IntPtr rowdata; + StringBuilder builder = new StringBuilder(); + + while ((rowdata = TDengine.FetchRows(res)) != IntPtr.Zero) + { + queryRows++; + for (int fields = 0; fields < fieldCount; ++fields) + { + TDengineMeta meta = metas[fields]; + int offset = IntPtr.Size * fields; + IntPtr data = Marshal.ReadIntPtr(rowdata, offset); + + builder.Append("---"); + + if (data == IntPtr.Zero) + { + builder.Append("NULL"); + continue; + } + + switch ((TDengineDataType)meta.type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + bool v1 = Marshal.ReadByte(data) == 0 ? false : true; + builder.Append(v1); + break; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + byte v2 = Marshal.ReadByte(data); + builder.Append(v2); + break; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + short v3 = Marshal.ReadInt16(data); + builder.Append(v3); + break; + case TDengineDataType.TSDB_DATA_TYPE_INT: + int v4 = Marshal.ReadInt32(data); + builder.Append(v4); + break; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + long v5 = Marshal.ReadInt64(data); + builder.Append(v5); + break; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); + builder.Append(v6); + break; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); + builder.Append(v7); + break; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + string v8 = Marshal.PtrToStringAnsi(data); + builder.Append(v8); + break; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + long v9 = Marshal.ReadInt64(data); + builder.Append(v9); + break; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + string v10 = Marshal.PtrToStringAnsi(data); + builder.Append(v10); + break; + } + } + builder.Append("---"); + + VerbosePrint(builder.ToString() + "\n"); + builder.Clear(); + } + + if (TDengine.ErrorNo(res) != 0) + { + Console.Write("Query is not complete, Error {0:G}", + TDengine.ErrorNo(res), TDengine.Error(res)); + } + + TDengine.FreeResult(res); + } + } + + public void CloseConnection() + { + if (this.conn != IntPtr.Zero) + { + TDengine.Close(this.conn); + } + } + + // Main entry + static void Main(string[] args) + { + PrintHelp(args); + + TDengineTest tester = new TDengineTest(); + tester.ReadArgument(args); + + tester.InitTDengine(); + tester.ConnectTDengine(); + + if (tester.isInsertOnly == false) + { + tester.dropDatabase(); + tester.CreateDb(); + + + if (tester.useStable == true) + { + tester.CreateStable(); + } + + tester.CreateTablesByThreads(); + } + + Stopwatch watch = Stopwatch.StartNew(); + tester.InsertByThreads(); + watch.Stop(); + double elapsedMs = watch.Elapsed.TotalMilliseconds; + + Console.WriteLine("C# taosdemo: Spent {0} seconds to insert {1} records with {2} record(s) per request: {3} records/second", + elapsedMs / 1000, + tester.recordsPerTable * tester.numOfTables, + tester.batchRows, + (tester.recordsPerTable * tester.numOfTables * 1000) / elapsedMs); + + tester.DebugPrintFormat("query command:{0}\n", tester.query); + if (tester.query != "NONE") + { + watch = Stopwatch.StartNew(); + tester.ExecuteQuery(); + watch.Stop(); + elapsedMs = watch.Elapsed.TotalMilliseconds; + Console.WriteLine("C# taosdemo: Spent {0} seconds to query {1} records.\n", + elapsedMs/1000, + tester.recordsPerTable * tester.numOfTables + ); + } + tester.CloseConnection(); + + Console.WriteLine("End."); + CleanAndExitProgram(0); + } + + public class InsertDataThread + { + public long id { set; get; } + public long start { set; get; } + public long end { set; get; } + public string dbName { set; get; } + public IntPtr conn { set; get; } + public string tablePrefix { set; get; } + public string stablePrefix { set; get; } + public long recordsPerTable { set; get; } + public long batchRows { set; get; } + public long numOfTables { set; get; } + public bool verbose { set; get; } + public bool debug { set; get; } + public bool order { set; get; } + public short rateOfOutorder { set; get; } + + private void VerbosePrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void VerbosePrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (debug == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (debug == true) + { + Console.Write(str); + } + } + + public void ThreadMain() + { + VerbosePrintFormat("InsertDataThread {0} from {1} to {2}\n", id, start, end); + StringBuilder sql = new StringBuilder(); + + DateTime now = DateTime.Now; + int h = now.Hour; + int m = now.Minute; + int s = now.Second; + + long baseTimestamp = 1609430400000; // 2021/01/01 0:0:0 + VerbosePrintFormat("beginTime is {0} + {1}h:{2}m:{3}s\n", baseTimestamp, h, m, s); + long beginTimestamp = baseTimestamp + ((h*60 + m) * 60 + s) * 1000; + Random random = new Random(); + + long rowsInserted = 0; + + long i = 0; + while (i < recordsPerTable) + { + for (long table = start; table <= end; ++table) + { + long inserted = i; + + sql.Clear(); + sql.Append("INSERT INTO "). + Append(this.dbName).Append(".").Append(this.tablePrefix).Append(table). + Append(" VALUES"); + if (recordsPerTable < batchRows) + { + batchRows = recordsPerTable; + } + for (int batch = 0; batch < batchRows; ++batch) + { + long writeTimeStamp = beginTimestamp + i + batch; + int rnd = 100; + if (this.order == false) + { + rnd = random.Next(1, 100); + if (rnd <= this.rateOfOutorder) + { + writeTimeStamp = writeTimeStamp + rnd * 10000; + DebugPrint("### "); + } + DebugPrintFormat("order:{0} rnd:{1} timestamp:{2}\n", this.order, rnd, writeTimeStamp); + } + else + { + DebugPrintFormat("order:{0} timestamp:{1}\n", this.order, writeTimeStamp); + } + + sql.Append("(") + .Append(writeTimeStamp) + .Append(", 1, 2, 3,") + .Append(i + batch) + .Append(", 5, 6, 7, 'abc', 'def')"); + + } + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res == IntPtr.Zero) + { + VerbosePrint(sql.ToString() + " failure, reason: " + TDengine.Error(res) + "\n"); + } + + inserted += this.batchRows; + + int affectRows = TDengine.AffectRows(res); + rowsInserted += affectRows; + + TDengine.FreeResult(res); + if (table == end) + { + i = inserted; + } + } + } + + } + } + + public class CreateTableThread + { + public long id { set; get; } + public long start { set; get; } + public long end { set; get; } + public string dbName { set; get; } + public IntPtr conn { set; get; } + public string tablePrefix { set; get; } + public string stablePrefix { set; get; } + public bool verbose { set; get; } + public bool debug { set; get; } + public bool useStable { set; get; } + + private void VerbosePrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void VerbosePrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (debug == true) + { + Console.Write(format, parameters); + } + } + + public void ThreadMain() + { + VerbosePrintFormat("CreateTable {0} from {1} to {2}\n", id, start, end); + + StringBuilder sql = new StringBuilder(); + + for (long tableId = start; tableId <= end; tableId++) + { + sql.Clear(); + sql = sql.Append("CREATE TABLE IF NOT EXISTS "). + Append(this.dbName).Append(".").Append(this.tablePrefix).Append(tableId); + if (useStable == true) + { + sql = sql.Append(" USING ").Append(this.dbName).Append(".").Append(this.stablePrefix). + Append(" TAGS(").Append(tableId).Append(")"); + } + else + { + sql = sql.Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10))"); + } + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + VerbosePrint(sql.ToString() + " success\n"); + } + else + { + VerbosePrint(sql.ToString() + " failure, reason: " + TDengine.Error(res) + "\n"); + CleanAndExitProgram(1); + } + TDengine.FreeResult(res); + } + + } + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md b/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md index e14a5f7b675fa1f38514f93afd853e7a1be2a77f..17c5c8df00ab8727d1adfe493d3fbbd32891a676 100644 --- a/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md +++ b/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md @@ -137,7 +137,7 @@ Welcome to the TDengine shell from Linux, Client Version:2.0.1.1 Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | + name | created_time | ntables | vgroups | replica | quorum | days | keep0,keep1,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | =================================================================================================================================================================================================================================================================== test | 2020-08-19 18:43:50.731 | 1 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | log | 2020-08-19 18:40:28.064 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index 98f908b77e8d3bfa03eebf8560205b0424720f1f..167e7e37ae410dd865aa2e08c7f885961f9ddadd 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -5,17 +5,20 @@ 4.0.0 com.taosdata.jdbc - jdbcChecker + JDBCDemo SNAPSHOT jar + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.20 + + + - - org.apache.maven.plugins - maven-assembly-plugin - 3.0.0 - - org.apache.maven.plugins maven-assembly-plugin @@ -23,7 +26,7 @@ - com.taosdata.example.JdbcChecker + com.taosdata.example.JDBCDemo @@ -47,27 +50,11 @@ 8 8 + UTF-8 + - - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.12 - - - log4j - log4j - 1.2.17 - - - junit - junit - 4.13.1 - test - - diff --git a/tests/examples/JDBC/JDBCDemo/readme.md b/tests/examples/JDBC/JDBCDemo/readme.md index e348e458fe938c4f2381c448f3c15e60af27040e..94844420859a2cc56842e95e4ac8c664c8ffa2e7 100644 --- a/tests/examples/JDBC/JDBCDemo/readme.md +++ b/tests/examples/JDBC/JDBCDemo/readme.md @@ -1,29 +1,37 @@ -# How to Run the JDBC Demo Code On A Linux OS +# How to Run the JDBC Demo Code On Linux OS TDengine's JDBC demo project is organized in a Maven way so that users can easily compile, package and run the project. If you don't have Maven on your server, you may install it using -
    sudo apt-get install maven
    +``` +sudo apt-get install maven +``` ## Install TDengine Client Make sure you have already installed a tdengine client on your current develop environment. Download the tdengine package on our website: ``https://www.taosdata.com/cn/all-downloads/`` and install the client. -## How to run jdbcChecker -
    mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcChecker" -Dexec.args="-host localhost"
    - -## How to run jdbcTaosDemo +## Run jdbcDemo using mvn plugin run command: -
     mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo"
    +``` +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JDBCDemo" +``` + and run with your customed args -
    mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo" -Dexec.args="-host localhost"
    +``` +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JDBCDemo" -Dexec.args="-host [HOSTNAME]" +``` ## Compile the Demo Code and Run It +To compile taos-jdbcdriver, go to the source directory ``TDengine/src/connector/jdbc`` and execute +``` +mvn clean package -Dmaven.test.skip=true +``` To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute - -
    +```
 mvn clean package assembly:single
-
    +``` -The ``pom.xml`` is configured to package all the dependencies into one executable jar file. +To run JDBCDemo.jar, go to ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute +``` +java -Djava.ext.dirs=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host [HOSTNAME] +``` -To run it, go to ``examples/JDBC/JDBCDemo/target`` and execute -
    java -jar jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host localhost
    diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcChecker.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java similarity index 60% rename from tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcChecker.java rename to tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java index 4be71c52214c348ed7b41c3e763de0d908514907..07a21e79d1cd188528fbbc7548e35fac78c65600 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcChecker.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java @@ -1,60 +1,95 @@ package com.taosdata.example; -import com.taosdata.jdbc.TSDBDriver; - import java.sql.*; import java.util.Properties; -public class JdbcChecker { +public class JDBCDemo { private static String host; - private static String dbName = "test"; - private static String tbName = "weather"; + private static final String dbName = "test"; + private static final String tbName = "weather"; private Connection connection; - /** - * get connection - **/ + public static void main(String[] args) { + for (int i = 0; i < args.length; i++) { + if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) + host = args[++i]; + } + if (host == null) { + printHelp(); + } + JDBCDemo demo = new JDBCDemo(); + demo.init(); + demo.createDatabase(); + demo.useDatabase(); + demo.dropTable(); + demo.createTable(); + demo.insert(); + demo.select(); + demo.dropTable(); + demo.close(); + } + private void init() { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + // get connection try { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + properties.setProperty("charset", "UTF-8"); + properties.setProperty("locale", "en_US.UTF-8"); + properties.setProperty("timezone", "UTC-8"); System.out.println("get connection starting..."); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + connection = DriverManager.getConnection(url, properties); if (connection != null) System.out.println("[ OK ] Connection established."); } catch (ClassNotFoundException | SQLException e) { - throw new RuntimeException("connection failed: " + host); + e.printStackTrace(); } } - /** - * create database - */ private void createDatabase() { String sql = "create database if not exists " + dbName; exuete(sql); } - /** - * use database - */ private void useDatabase() { String sql = "use " + dbName; exuete(sql); } - /** - * select - */ - private void checkSelect() { + private void dropTable() { + final String sql = "drop table if exists " + dbName + "." + tbName + ""; + exuete(sql); + } + + private void createTable() { + final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int)"; + exuete(sql); + } + + private void insert() { + final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; + exuete(sql); + } + + private void select() { final String sql = "select * from test.weather"; executeQuery(sql); } + private void close() { + try { + if (connection != null) { + this.connection.close(); + System.out.println("connection closed."); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /************************************************************************/ + private void executeQuery(String sql) { try (Statement statement = connection.createStatement()) { long start = System.currentTimeMillis(); @@ -79,40 +114,12 @@ public class JdbcChecker { } } - private String formatString(String str) { - StringBuilder sb = new StringBuilder(); - int blankCnt = (26 - str.length()) / 2; - for (int j = 0; j < blankCnt; j++) - sb.append(" "); - sb.append(str); - for (int j = 0; j < blankCnt; j++) - sb.append(" "); - sb.append("|"); - return sb.toString(); - } - - - /** - * insert - */ - private void checkInsert() { - final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; - exuete(sql); - } - - /** - * create table - */ - private void createTable() { - final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int)"; - exuete(sql); - } - private final void printSql(String sql, boolean succeed, long cost) { + private void printSql(String sql, boolean succeed, long cost) { System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); } - private final void exuete(String sql) { + private void exuete(String sql) { try (Statement statement = connection.createStatement()) { long start = System.currentTimeMillis(); boolean execute = statement.execute(sql); @@ -120,54 +127,14 @@ public class JdbcChecker { printSql(sql, execute, (end - start)); } catch (SQLException e) { e.printStackTrace(); - - } - } - private void close() { - try { - if (connection != null) { - this.connection.close(); - System.out.println("connection closed."); - } - } catch (SQLException e) { - e.printStackTrace(); } } - private void checkDropTable() { - final String sql = "drop table if exists " + dbName + "." + tbName + ""; - exuete(sql); + private static void printHelp() { + System.out.println("Usage: java -jar JDBCDemo.jar -host "); + System.exit(0); } - public static void main(String[] args) { - for (int i = 0; i < args.length; i++) { - if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { - host = args[++i]; - } - if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { - dbName = args[++i]; - } - if ("-t".equalsIgnoreCase(args[i]) && i < args.length - 1) { - tbName = args[++i]; - } - } - - if (host == null) { - System.out.println("Usage: java -jar JDBCConnectorChecker.jar -host "); - return; - } - - JdbcChecker checker = new JdbcChecker(); - checker.init(); - checker.createDatabase(); - checker.useDatabase(); - checker.checkDropTable(); - checker.createTable(); - checker.checkInsert(); - checker.checkSelect(); - checker.checkDropTable(); - checker.close(); - } } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java new file mode 100644 index 0000000000000000000000000000000000000000..eedb0ba166da50f21577296ad6fa1cd3f838ac32 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java @@ -0,0 +1,47 @@ +package com.taosdata.example; + +import java.sql.*; +import java.util.Properties; + +public class JdbcRestfulDemo { + private static final String host = "master"; + + public static void main(String[] args) { + try { + // load JDBC-restful driver + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + // use port 6041 in url when use JDBC-restful + String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + + Properties properties = new Properties(); + properties.setProperty("charset", "UTF-8"); + properties.setProperty("locale", "en_US.UTF-8"); + properties.setProperty("timezone", "UTC-8"); + + Connection conn = DriverManager.getConnection(url, properties); + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists restful_test"); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("create table restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"); + stmt.executeUpdate("insert into t1 using restful_test.weather tags('北京') values(now, 18.2)"); + ResultSet rs = stmt.executeQuery("select * from restful_test.weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + + rs.close(); + stmt.close(); + conn.close(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java new file mode 100644 index 0000000000000000000000000000000000000000..def4c649027034028d222bfedb71e37d82b99380 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -0,0 +1,75 @@ +package com.taosdata.example; + +import com.taosdata.jdbc.TSDBConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBResultSet; +import com.taosdata.jdbc.TSDBSubscribe; + +import java.sql.DriverManager; +import java.sql.ResultSetMetaData; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class SubscribeDemo { + private static final String usage = "java -jar SubscribeDemo.jar -host -database -topic -sql "; + + public static void main(String[] args) { + // parse args from command line + String host = "", database = "", topic = "", sql = ""; + for (int i = 0; i < args.length; i++) { + if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-database".equalsIgnoreCase(args[i]) && i < args.length - 1) { + database = args[++i]; + } + if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { + topic = args[++i]; + } + if ("-sql".equalsIgnoreCase(args[i]) && i < args.length - 1) { + sql = args[++i]; + } + } + if (host.isEmpty() || database.isEmpty() || topic.isEmpty() || sql.isEmpty()) { + System.out.println(usage); + return; + } + /*********************************************************************************************/ + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + final String url = "jdbc:TAOS://" + host + ":6030/" + database + "?user=root&password=taosdata"; + // get TSDBConnection + TSDBConnection connection = (TSDBConnection) DriverManager.getConnection(url, properties); + // create TSDBSubscribe + TSDBSubscribe sub = connection.subscribe(topic, sql, false); + + int total = 0; + while (true) { + TSDBResultSet rs = sub.consume(); + int count = 0; + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + count++; + } + total += count; +// System.out.printf("%d rows consumed, total %d\n", count, total); + if (total >= 10) + break; + TimeUnit.SECONDS.sleep(1); + } + sub.close(false); + connection.close(); + } catch (Exception e) { + System.out.println("host: " + host + ", database: " + database + ", topic: " + topic + ", sql: " + sql); + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java deleted file mode 100644 index 259985ec9f4708b9317575fd97919adcc82d7161..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java +++ /dev/null @@ -1,357 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo; - -import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; -import com.taosdata.example.jdbcTaosdemo.task.CreateTableTask; -import com.taosdata.example.jdbcTaosdemo.task.InsertTableDatetimeTask; -import com.taosdata.example.jdbcTaosdemo.task.InsertTableTask; -import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; -import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; -import com.taosdata.example.jdbcTaosdemo.utils.TimeStampUtil; -import org.apache.log4j.Logger; - -import java.sql.*; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class JdbcTaosdemo { - - private static Logger logger = Logger.getLogger(JdbcTaosdemo.class); - private final JdbcTaosdemoConfig config; - private Connection connection; - - public JdbcTaosdemo(JdbcTaosdemoConfig config) { - this.config = config; - } - - public static void main(String[] args) { - // parse config from args - JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); - - boolean isHelp = Arrays.asList(args).contains("--help"); - if (isHelp) { - JdbcTaosdemoConfig.printHelp(); - return; - } - if (config.getHost() == null) { - JdbcTaosdemoConfig.printHelp(); - return; - } - - JdbcTaosdemo taosdemo = new JdbcTaosdemo(config); - // establish connection - taosdemo.init(); - // drop database - taosdemo.dropDatabase(); - // create database - taosdemo.createDatabase(); - // use db - taosdemo.useDatabase(); - // create super table - taosdemo.createSuperTable(); - // create sub tables - taosdemo.createTableMultiThreads(); - - boolean infinite = Arrays.asList(args).contains("--infinite"); - if (infinite) { - logger.info("!!! Infinite Insert Mode Started. !!!"); - taosdemo.insertInfinite(); - } else { - // insert into table - taosdemo.insertMultiThreads(); - // select from sub table - taosdemo.selectFromTableLimit(); - taosdemo.selectCountFromTable(); - taosdemo.selectAvgMinMaxFromTable(); - // select last from - taosdemo.selectLastFromTable(); - // select from super table - taosdemo.selectFromSuperTableLimit(); - taosdemo.selectCountFromSuperTable(); - taosdemo.selectAvgMinMaxFromSuperTable(); - //select avg ,max from stb where tag - taosdemo.selectAvgMinMaxFromSuperTableWhereTag(); - //select last from stb where location = '' - taosdemo.selectLastFromSuperTableWhere(); - // select group by - taosdemo.selectGroupBy(); - // select like - taosdemo.selectLike(); - // select where ts >= ts<= - taosdemo.selectLastOneHour(); - taosdemo.selectLastOneDay(); - taosdemo.selectLastOneWeek(); - taosdemo.selectLastOneMonth(); - taosdemo.selectLastOneYear(); - - // drop super table - if (config.isDeleteTable()) - taosdemo.dropSuperTable(); - taosdemo.close(); - } - } - - - /** - * establish the connection - */ - private void init() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - connection = ConnectionFactory.build(config); - if (connection != null) - logger.info("[ OK ] Connection established."); - } catch (ClassNotFoundException | SQLException e) { - logger.error(e.getMessage()); - throw new RuntimeException("connection failed: " + config.getHost()); - } - } - - /** - * create database - */ - private void createDatabase() { - String sql = SqlSpeller.createDatabaseSQL(config.getDbName(), config.getKeep(), config.getDays()); - execute(sql); - } - - /** - * drop database - */ - private void dropDatabase() { - String sql = SqlSpeller.dropDatabaseSQL(config.getDbName()); - execute(sql); - } - - /** - * use database - */ - private void useDatabase() { - String sql = SqlSpeller.useDatabaseSQL(config.getDbName()); - execute(sql); - } - - /** - * create super table - */ - private void createSuperTable() { - String sql = SqlSpeller.createSuperTableSQL(config.getStbName()); - execute(sql); - } - - /** - * create table use super table with multi threads - */ - private void createTableMultiThreads() { - try { - final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); - List threads = new ArrayList<>(); - for (int i = 0; i < config.getNumberOfThreads(); i++) { - Thread thread = new Thread(new CreateTableTask(config, i * tableSize, tableSize), "Thread-" + i); - threads.add(thread); - thread.start(); - } - for (Thread thread : threads) { - thread.join(); - } - logger.info("<<< Multi Threads create table finished."); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - e.printStackTrace(); - } - } - - /** - * insert data infinitely - */ - private void insertInfinite() { - try { - final long startDatetime = TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000"); - final long finishDatetime = TimeStampUtil.datetimeToLong("2030-01-01 00:00:00.000"); - - final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); - List threads = new ArrayList<>(); - for (int i = 0; i < config.getNumberOfThreads(); i++) { - Thread thread = new Thread(new InsertTableDatetimeTask(config, i * tableSize, tableSize, startDatetime, finishDatetime), "Thread-" + i); - threads.add(thread); - thread.start(); - } - for (Thread thread : threads) { - thread.join(); - } - logger.info("<<< Multi Threads insert table finished."); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - e.printStackTrace(); - } - } - - private void insertMultiThreads() { - try { - final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); - final int numberOfRecordsPerTable = config.getNumberOfRecordsPerTable(); - List threads = new ArrayList<>(); - for (int i = 0; i < config.getNumberOfThreads(); i++) { - Thread thread = new Thread(new InsertTableTask(config, i * tableSize, tableSize, numberOfRecordsPerTable), "Thread-" + i); - threads.add(thread); - thread.start(); - } - for (Thread thread : threads) { - thread.join(); - } - logger.info("<<< Multi Threads insert table finished."); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - e.printStackTrace(); - } - } - - private void selectFromTableLimit() { - String sql = SqlSpeller.selectFromTableLimitSQL(config.getDbName(), config.getTbPrefix(), 1, 10, 0); - executeQuery(sql); - } - - private void selectCountFromTable() { - String sql = SqlSpeller.selectCountFromTableSQL(config.getDbName(), config.getTbPrefix(), 1); - executeQuery(sql); - } - - private void selectAvgMinMaxFromTable() { - String sql = SqlSpeller.selectAvgMinMaxFromTableSQL("current", config.getDbName(), config.getTbPrefix(), 1); - executeQuery(sql); - } - - private void selectLastFromTable() { - String sql = SqlSpeller.selectLastFromTableSQL(config.getDbName(), config.getTbPrefix(), 1); - executeQuery(sql); - } - - private void selectFromSuperTableLimit() { - String sql = SqlSpeller.selectFromSuperTableLimitSQL(config.getDbName(), config.getStbName(), 10, 0); - executeQuery(sql); - } - - private void selectCountFromSuperTable() { - String sql = SqlSpeller.selectCountFromSuperTableSQL(config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectAvgMinMaxFromSuperTable() { - String sql = SqlSpeller.selectAvgMinMaxFromSuperTableSQL("current", config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectAvgMinMaxFromSuperTableWhereTag() { - String sql = SqlSpeller.selectAvgMinMaxFromSuperTableWhere("current", config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectLastFromSuperTableWhere() { - String sql = SqlSpeller.selectLastFromSuperTableWhere("current", config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectGroupBy() { - String sql = SqlSpeller.selectGroupBy("current", config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectLike() { - String sql = SqlSpeller.selectLike(config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectLastOneHour() { - String sql = SqlSpeller.selectLastOneHour(config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectLastOneDay() { - String sql = SqlSpeller.selectLastOneDay(config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectLastOneWeek() { - String sql = SqlSpeller.selectLastOneWeek(config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectLastOneMonth() { - String sql = SqlSpeller.selectLastOneMonth(config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - private void selectLastOneYear() { - String sql = SqlSpeller.selectLastOneYear(config.getDbName(), config.getStbName()); - executeQuery(sql); - } - - - private void close() { - try { - if (connection != null) { - this.connection.close(); - logger.info("connection closed."); - } - } catch (SQLException e) { - logger.error(e.getMessage()); - e.printStackTrace(); - } - } - - /** - * drop super table - */ - private void dropSuperTable() { - String sql = SqlSpeller.dropSuperTableSQL(config.getDbName(), config.getStbName()); - execute(sql); - } - - /** - * execute sql, use this method when sql is create, alter, drop.. - */ - private void execute(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - boolean execute = statement.execute(sql); - long end = System.currentTimeMillis(); - printSql(sql, execute, (end - start)); - } catch (SQLException e) { - logger.error("ERROR execute SQL ===> " + sql); - logger.error(e.getMessage()); - e.printStackTrace(); - } - } - - private static void printSql(String sql, boolean succeed, long cost) { - System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); - } - - private void executeQuery(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - ResultSet resultSet = statement.executeQuery(sql); - long end = System.currentTimeMillis(); - printSql(sql, true, (end - start)); - printResult(resultSet); - } catch (SQLException e) { - logger.error("ERROR execute SQL ===> " + sql); - logger.error(e.getMessage()); - e.printStackTrace(); - } - } - - private static void printResult(ResultSet resultSet) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - StringBuilder sb = new StringBuilder(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String columnLabel = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - sb.append(columnLabel + ": " + value + "\t"); - } - System.out.println(sb.toString()); - } - } - -} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java deleted file mode 100644 index 36745a93941cc690f37d06d9a3662605723bbd2c..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.domain; - -public final class JdbcTaosdemoConfig { - - //The host to connect to TDengine. Must insert one - private String host; - //The TCP/IP port number to use for the connection. Default is 6030. - private int port = 6030; - //The TDengine user name to use when connecting to the server. Default is 'root' - private String user = "root"; - //The password to use when connecting to the server. Default is 'taosdata' - private String password = "taosdata"; - - //Destination database. Default is 'test' - private String dbName = "test"; - //keep - private int keep = 36500; - //days - private int days = 120; - - //Super table Name. Default is 'meters' - private String stbName = "meters"; - //Table name prefix. Default is 'd' - private String tbPrefix = "d"; - //The number of tables. Default is 10. - private int numberOfTable = 10; - //The number of records per table. Default is 2 - private int numberOfRecordsPerTable = 2; - //The number of records per request. Default is 100 - private int numberOfRecordsPerRequest = 100; - - //The number of threads. Default is 1. - private int numberOfThreads = 1; - //Delete data. Default is false - private boolean deleteTable = false; - - public static void printHelp() { - System.out.println("Usage: java -jar JdbcTaosDemo.jar [OPTION...]"); - System.out.println("-h host The host to connect to TDengine. you must input one"); - System.out.println("-p port The TCP/IP port number to use for the connection. Default is 6030"); - System.out.println("-u user The TDengine user name to use when connecting to the server. Default is 'root'"); - System.out.println("-P password The password to use when connecting to the server.Default is 'taosdata'"); - System.out.println("-d database Destination database. Default is 'test'"); - System.out.println("-m tablePrefix Table prefix name. Default is 'd'"); - System.out.println("-t num_of_tables The number of tables. Default is 10"); - System.out.println("-n num_of_records_per_table The number of records per table. Default is 2"); - System.out.println("-r num_of_records_per_req The number of records per request. Default is 100"); - System.out.println("-T num_of_threads The number of threads. Default is 1"); - System.out.println("-D delete table Delete data methods. Default is false"); - System.out.println("--help Give this help list"); -// System.out.println("--infinite infinite insert mode"); - } - - /** - * parse args from command line - * - * @param args command line args - * @return JdbcTaosdemoConfig - */ - public JdbcTaosdemoConfig(String[] args) { - for (int i = 0; i < args.length; i++) { - if ("-h".equals(args[i]) && i < args.length - 1) { - host = args[++i]; - } - if ("-p".equals(args[i]) && i < args.length - 1) { - port = Integer.parseInt(args[++i]); - } - if ("-u".equals(args[i]) && i < args.length - 1) { - user = args[++i]; - } - if ("-P".equals(args[i]) && i < args.length - 1) { - password = args[++i]; - } - if ("-d".equals(args[i]) && i < args.length - 1) { - dbName = args[++i]; - } - if ("-m".equals(args[i]) && i < args.length - 1) { - tbPrefix = args[++i]; - } - if ("-t".equals(args[i]) && i < args.length - 1) { - numberOfTable = Integer.parseInt(args[++i]); - } - if ("-n".equals(args[i]) && i < args.length - 1) { - numberOfRecordsPerTable = Integer.parseInt(args[++i]); - } - if ("-r".equals(args[i]) && i < args.length - 1) { - numberOfRecordsPerRequest = Integer.parseInt(args[++i]); - } - if ("-T".equals(args[i]) && i < args.length - 1) { - numberOfThreads = Integer.parseInt(args[++i]); - } - if ("-D".equals(args[i]) && i < args.length - 1) { - deleteTable = Boolean.parseBoolean(args[++i]); - } - } - } - - public String getHost() { - return host; - } - - public int getPort() { - return port; - } - - public String getUser() { - return user; - } - - public String getPassword() { - return password; - } - - public String getDbName() { - return dbName; - } - - public int getKeep() { - return keep; - } - - public int getDays() { - return days; - } - - public String getStbName() { - return stbName; - } - - public String getTbPrefix() { - return tbPrefix; - } - - public int getNumberOfTable() { - return numberOfTable; - } - - public int getNumberOfRecordsPerTable() { - return numberOfRecordsPerTable; - } - - public int getNumberOfThreads() { - return numberOfThreads; - } - - public boolean isDeleteTable() { - return deleteTable; - } - - public int getNumberOfRecordsPerRequest() { - return numberOfRecordsPerRequest; - } -} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java deleted file mode 100644 index 1da2c8647efe0e9204f1a591ba9431a489c91cb0..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.task; - -import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; -import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; -import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; -import org.apache.log4j.Logger; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; - -public class CreateTableTask implements Runnable { - - private static Logger logger = Logger.getLogger(CreateTableTask.class); - private final JdbcTaosdemoConfig config; - private final int startIndex; - private final int tableNumber; - - public CreateTableTask(JdbcTaosdemoConfig config, int startIndex, int tableNumber) { - this.config = config; - this.startIndex = startIndex; - this.tableNumber = tableNumber; - } - - @Override - public void run() { - try { - Connection connection = ConnectionFactory.build(config); - for (int i = startIndex; i < startIndex + tableNumber; i++) { - Statement statement = connection.createStatement(); - String sql = SqlSpeller.createTableSQL(i + 1, config.getDbName(), config.getStbName()); - statement.execute(sql); - statement.close(); - logger.info(">>> " + sql); - } - connection.close(); - } catch (SQLException e) { - logger.error(e.getMessage()); - e.printStackTrace(); - } - } -} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java deleted file mode 100644 index 4f60c25646573223a9cbfd820c8eb37e4f6f6c8c..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.task; - -import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; -import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; -import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; -import org.apache.log4j.Logger; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; - -public class InsertTableDatetimeTask implements Runnable { - private static Logger logger = Logger.getLogger(InsertTableDatetimeTask.class); - - private final JdbcTaosdemoConfig config; - private final int startTableIndex; - private final int tableNumber; - private final long startDatetime; - private final long finishedDatetime; - - public InsertTableDatetimeTask(JdbcTaosdemoConfig config, int startTableIndex, int tableNumber, long startDatetime, long finishedDatetime) { - this.config = config; - this.startTableIndex = startTableIndex; - this.tableNumber = tableNumber; - this.startDatetime = startDatetime; - this.finishedDatetime = finishedDatetime; - } - - @Override - public void run() { - try { - Connection connection = ConnectionFactory.build(config); - int valuesCount = config.getNumberOfRecordsPerRequest(); - for (long ts = startDatetime; ts < finishedDatetime; ts += valuesCount) { - for (int i = startTableIndex; i < startTableIndex + tableNumber; i++) { - String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, valuesCount); - Statement statement = connection.createStatement(); - statement.execute(sql); - statement.close(); - logger.info(Thread.currentThread().getName() + ">>> " + sql); - } - } - connection.close(); - } catch (SQLException e) { - logger.error(e.getMessage()); - e.printStackTrace(); - } - } -} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java deleted file mode 100644 index 644de52dd3e75a77c2d635a6f5328f186259096c..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.task; - -import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; -import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory; -import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller; -import org.apache.log4j.Logger; - -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -public class InsertTableTask implements Runnable { - private static final Logger logger = Logger.getLogger(InsertTableTask.class); - - private final JdbcTaosdemoConfig config; - private final int startTbIndex; - private final int tableNumber; - private final int recordsNumberPerTable; - - public InsertTableTask(JdbcTaosdemoConfig config, int startTbIndex, int tableNumber, int recordsNumberPerTable) { - this.config = config; - this.startTbIndex = startTbIndex; - this.tableNumber = tableNumber; - this.recordsNumberPerTable = recordsNumberPerTable; - } - - @Override - public void run() { - try { - Connection connection = ConnectionFactory.build(config); - int keep = config.getKeep(); - Instant end = Instant.now(); - Instant start = end.minus(Duration.ofDays(keep - 1)); - long timeGap = ChronoUnit.MILLIS.between(start, end) / (recordsNumberPerTable - 1); - - // iterate insert - for (int j = 0; j < recordsNumberPerTable; j++) { - long ts = start.toEpochMilli() + (j * timeGap); - // insert data into echo table - for (int i = startTbIndex; i < startTbIndex + tableNumber; i++) { - String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, config.getNumberOfRecordsPerRequest()); - logger.info(Thread.currentThread().getName() + ">>> " + sql); - Statement statement = connection.createStatement(); - statement.execute(sql); - statement.close(); - } - } - connection.close(); - } catch (SQLException e) { - logger.error(e.getMessage()); - e.printStackTrace(); - } - } -} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java deleted file mode 100644 index 52691f4de722db2eb17e7061e09ff0e59a390077..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.utils; - -import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig; -import com.taosdata.jdbc.TSDBDriver; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Properties; - -public class ConnectionFactory { - - public static Connection build(JdbcTaosdemoConfig config) throws SQLException { - return build(config.getHost(), config.getPort(), config.getDbName(), config.getUser(), config.getPassword()); - } - - public static Connection build(String host, int port, String dbName) throws SQLException { - return build(host, port, dbName, "root", "taosdata"); - } - - private static Connection build(String host, int port, String dbName, String user, String password) throws SQLException { - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, user); - properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, password); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - return DriverManager.getConnection("jdbc:TAOS://" + host + ":" + port + "/" + dbName + "", properties); - } - - -} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java deleted file mode 100644 index b4a79e9eba47cc947d822b645d0ae1f9952f08f0..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java +++ /dev/null @@ -1,126 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.utils; - -import java.util.Random; - -public class SqlSpeller { - private static final Random random = new Random(System.currentTimeMillis()); - private static final String[] locations = { - "Beijing", "Shanghai", "Guangzhou", "Shenzhen", - "HangZhou", "Tianjin", "Wuhan", "Changsha", "Nanjing", "Xian" - }; - - public static String createDatabaseSQL(String dbName, int keep, int days) { - return "create database if not exists " + dbName + " keep " + keep + " days " + days; - } - - public static String dropDatabaseSQL(String dbName) { - return "drop database if exists " + dbName; - } - - public static String useDatabaseSQL(String dbName) { - return "use " + dbName; - } - - public static String createSuperTableSQL(String superTableName) { - return "create table if not exists " + superTableName + "(ts timestamp, current float, voltage int, phase float) tags(location binary(64), groupId int)"; - } - - public static String dropSuperTableSQL(String dbName, String superTableName) { - return "drop table if exists " + dbName + "." + superTableName; - } - - public static String createTableSQL(int tableIndex, String dbName, String superTableName) { - String location = locations[random.nextInt(locations.length)]; - return "create table d" + tableIndex + " using " + dbName + "." + superTableName + " tags('" + location + "'," + tableIndex + ")"; - } - - public static String insertOneRowSQL(String dbName, String tbPrefix, int tableIndex, long ts) { - float current = 10 + random.nextFloat(); - int voltage = 200 + random.nextInt(20); - float phase = random.nextFloat(); - String sql = "insert into " + dbName + "." + tbPrefix + "" + tableIndex + " " + "values(" + ts + ", " + current + ", " + voltage + ", " + phase + ")"; - return sql; - } - - public static String insertBatchSizeRowsSQL(String dbName, String tbPrefix, int tbIndex, long ts, int valuesCount) { - float current = 10 + random.nextFloat(); - int voltage = 200 + random.nextInt(20); - float phase = random.nextFloat(); - StringBuilder sb = new StringBuilder(); - sb.append("insert into " + dbName + "." + tbPrefix + "" + tbIndex + " " + "values"); - for (int i = 0; i < valuesCount; i++) { - sb.append("(" + (ts + i) + ", " + current + ", " + voltage + ", " + phase + ") "); - } - return sb.toString(); - } - - public static String selectFromTableLimitSQL(String dbName, String tbPrefix, int tbIndex, int limit, int offset) { - return "select * from " + dbName + "." + tbPrefix + "" + tbIndex + " limit " + limit + " offset " + offset; - } - - public static String selectCountFromTableSQL(String dbName, String tbPrefix, int tbIndex) { - return "select count(*) from " + dbName + "." + tbPrefix + "" + tbIndex; - } - - public static String selectAvgMinMaxFromTableSQL(String field, String dbName, String tbPrefix, int tbIndex) { - return "select avg(" + field + "),min(" + field + "),max(" + field + ") from " + dbName + "." + tbPrefix + "" + tbIndex; - } - - public static String selectFromSuperTableLimitSQL(String dbName, String stbName, int limit, int offset) { - return "select * from " + dbName + "." + stbName + " limit " + limit + " offset " + offset; - } - - public static String selectCountFromSuperTableSQL(String dbName, String stableName) { - return "select count(*) from " + dbName + "." + stableName; - } - - public static String selectAvgMinMaxFromSuperTableSQL(String field, String dbName, String stbName) { - return "select avg(" + field + "),min(" + field + "),max(" + field + ") from " + dbName + "." + stbName + ""; - } - - public static String selectLastFromTableSQL(String dbName, String tbPrefix, int tbIndex) { - return "select last(*) from " + dbName + "." + tbPrefix + "" + tbIndex; - } - - //select avg ,max from stb where tag - public static String selectAvgMinMaxFromSuperTableWhere(String field, String dbName, String stbName) { - return "select avg(" + field + "),min(" + field + "),max(" + field + ") from " + dbName + "." + stbName + " where location = '" + locations[random.nextInt(locations.length)] + "'"; - } - - //select last from stb where - public static String selectLastFromSuperTableWhere(String field, String dbName, String stbName) { - return "select last(" + field + ") from " + dbName + "." + stbName + " where location = '" + locations[random.nextInt(locations.length)] + "'"; - } - - public static String selectGroupBy(String field, String dbName, String stbName) { - return "select avg(" + field + ") from " + dbName + "." + stbName + " group by location"; - } - - public static String selectLike(String dbName, String stbName) { - return "select * from " + dbName + "." + stbName + " where location like 'S%'"; - } - - public static String selectLastOneHour(String dbName, String stbName) { - return "select * from " + dbName + "." + stbName + " where ts >= now - 1h"; - } - - public static String selectLastOneDay(String dbName, String stbName) { - return "select * from " + dbName + "." + stbName + " where ts >= now - 1d"; - } - - public static String selectLastOneWeek(String dbName, String stbName) { - return "select * from " + dbName + "." + stbName + " where ts >= now - 1w"; - } - - public static String selectLastOneMonth(String dbName, String stbName) { - return "select * from " + dbName + "." + stbName + " where ts >= now - 1n"; - } - - public static String selectLastOneYear(String dbName, String stbName) { - return "select * from " + dbName + "." + stbName + " where ts >= now - 1y"; - } - - // select group by - // select like - // select ts >= ts<= -} \ No newline at end of file diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtil.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtil.java deleted file mode 100644 index 0a345afdd1e45123d889d7ee198cf8efd201176b..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtil.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.utils; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.time.Duration; -import java.time.Instant; -import java.util.Date; - -public class TimeStampUtil { - private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; - - public static long datetimeToLong(String dateTime) { - SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); - try { - return sdf.parse(dateTime).getTime(); - } catch (ParseException e) { - throw new RuntimeException(e); - } - } - - public static String longToDatetime(long time) { - SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); - return sdf.format(new Date(time)); - } - - public static void main(String[] args) throws ParseException { - -// Instant now = Instant.now(); -// System.out.println(now); -// Instant years20Ago = now.minus(Duration.ofDays(365)); -// System.out.println(years20Ago); - - - } - - -} diff --git a/tests/examples/JDBC/JDBCDemo/src/test/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtilTest.java b/tests/examples/JDBC/JDBCDemo/src/test/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtilTest.java deleted file mode 100644 index f370b2ef6eaa708b061ebf4a7f58f3d31f78f999..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/JDBCDemo/src/test/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtilTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.taosdata.example.jdbcTaosdemo.utils; - -import org.junit.Test; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.time.Duration; -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.Date; - -import static org.junit.Assert.*; - -public class TimeStampUtilTest { - - @Test - public void datetimeToLong() { - final String startTime = "2005-01-01 00:00:00.000"; - long start = TimeStampUtil.datetimeToLong(startTime); - assertEquals(1104508800000l, start); - } - - @Test - public void longToDatetime() { - String datetime = TimeStampUtil.longToDatetime(1510000000000L); - assertEquals("2017-11-07 04:26:40.000", datetime); - } - - @Test - public void getStartDateTime() { - int keep = 365; - - Instant end = Instant.now(); - System.out.println(end.toString()); - System.out.println(end.toEpochMilli()); - - Instant start = end.minus(Duration.ofDays(keep)); - System.out.println(start.toString()); - System.out.println(start.toEpochMilli()); - - int numberOfRecordsPerTable = 10; - long timeGap = ChronoUnit.MILLIS.between(start, end) / (numberOfRecordsPerTable - 1); - System.out.println(timeGap); - - System.out.println("==========================="); - for (int i = 0; i < numberOfRecordsPerTable; i++) { - long ts = start.toEpochMilli() + (i * timeGap); - System.out.println(i + " : " + ts); - } - } -} \ No newline at end of file diff --git a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml index 15aed1cf03b4e10e5a69e16be99ea2320a24609e..64a91b951bafe65e2f685fb57091221760fb99f9 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.18 @@ -69,7 +69,7 @@ - com.taosdata.jdbc.example.jdbcTemplate.App + com.taosdata.example.jdbcTemplate.App diff --git a/tests/examples/JDBC/SpringJdbcTemplate/readme.md b/tests/examples/JDBC/SpringJdbcTemplate/readme.md index 1fe8809b506c248f226edd0f3200c6e352c0a73b..b70a6565f88d0a08b8a26a60676e729ecdb39e2e 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/readme.md +++ b/tests/examples/JDBC/SpringJdbcTemplate/readme.md @@ -8,18 +8,16 @@ 修改 `src/main/resources/applicationContext.xml` 文件中 TDengine 的配置信息： ```xml - - - - - - - - - - - - + + + + + + + + + + ``` ### 打包运行 diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/App.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java similarity index 86% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/App.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java index a03ca3924f9d86199c461a21d887b78e433ef6fe..6942d62a83adafb85496a81ce93866cd0d53611d 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/App.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java @@ -1,9 +1,9 @@ -package com.taosdata.jdbc.example.jdbcTemplate; +package com.taosdata.example.jdbcTemplate; -import com.taosdata.jdbc.example.jdbcTemplate.dao.ExecuteAsStatement; -import com.taosdata.jdbc.example.jdbcTemplate.dao.WeatherDao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.dao.ExecuteAsStatement; +import com.taosdata.example.jdbcTemplate.dao.WeatherDao; +import com.taosdata.example.jdbcTemplate.domain.Weather; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/ExecuteAsStatement.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatement.java similarity index 58% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/ExecuteAsStatement.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatement.java index f146684cc01f75a211182a90e5cfee4058aea413..5947438e408b3b359b138bf989477f60fc327404 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/ExecuteAsStatement.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatement.java @@ -1,4 +1,4 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao; +package com.taosdata.example.jdbcTemplate.dao; public interface ExecuteAsStatement{ diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/ExecuteAsStatementImpl.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatementImpl.java similarity index 75% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/ExecuteAsStatementImpl.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatementImpl.java index 2700e701cc42034975ff902ebe1da01f040b5a69..059e3dda15ea36bcaad4309a434a06f020d0301b 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/ExecuteAsStatementImpl.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatementImpl.java @@ -1,6 +1,5 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao.impl; +package com.taosdata.example.jdbcTemplate.dao; -import com.taosdata.jdbc.example.jdbcTemplate.dao.ExecuteAsStatement; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/WeatherDao.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDao.java similarity index 65% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/WeatherDao.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDao.java index 28962ee1e6803898a0ec24e2231a62d91bcbf6d6..19a07597f8df869dada4e258c4951f67132107a0 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/WeatherDao.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDao.java @@ -1,6 +1,6 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao; +package com.taosdata.example.jdbcTemplate.dao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.domain.Weather; import java.util.List; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/WeatherDaoImpl.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDaoImpl.java similarity index 84% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/WeatherDaoImpl.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDaoImpl.java index 1e0e0ab68c63bdb87ad172f277fe2a65df79d229..8d4ca47d5ee5d351a48bcc1554531566116f5a52 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/WeatherDaoImpl.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDaoImpl.java @@ -1,20 +1,16 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao.impl; +package com.taosdata.example.jdbcTemplate.dao; -import com.taosdata.jdbc.example.jdbcTemplate.dao.WeatherDao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.dao.WeatherDao; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.namedparam.SqlParameterSourceUtils; -import org.springframework.jdbc.core.simple.SimpleJdbcInsert; import org.springframework.stereotype.Repository; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.HashMap; import java.util.List; -import java.util.Map; @Repository public class WeatherDaoImpl implements WeatherDao { diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/domain/Weather.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/domain/Weather.java similarity index 94% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/domain/Weather.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/domain/Weather.java index 023b301481829b8d73a8af566d418aa8232ef899..1787a08c3554e017645fefbd35d7c4fd1bc60bd4 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/domain/Weather.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/domain/Weather.java @@ -1,4 +1,4 @@ -package com.taosdata.jdbc.example.jdbcTemplate.domain; +package com.taosdata.example.jdbcTemplate.domain; import java.sql.Timestamp; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml b/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml index 19ac433385fabe53548dcba25d5bb10965df17af..6d6cf6047e2545256ffd6f39e1e6160a965a6fa8 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml @@ -10,7 +10,7 @@ - + @@ -20,6 +20,6 @@ - + diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/example/jdbcTemplate/BatcherInsertTest.java b/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java similarity index 89% rename from tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/example/jdbcTemplate/BatcherInsertTest.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java index 2f2446eb7049ae44ff3e142d62a2e51ab71b6f16..29d0f79fd4982d43078e590b4320c0df457ee44c 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/example/jdbcTemplate/BatcherInsertTest.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java @@ -1,9 +1,9 @@ -package com.taosdata.jdbc.example.jdbcTemplate; +package com.taosdata.example.jdbcTemplate; -import com.taosdata.jdbc.example.jdbcTemplate.dao.ExecuteAsStatement; -import com.taosdata.jdbc.example.jdbcTemplate.dao.WeatherDao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.dao.ExecuteAsStatement; +import com.taosdata.example.jdbcTemplate.dao.WeatherDao; +import com.taosdata.example.jdbcTemplate.domain.Weather; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/AppTest.java b/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/AppTest.java deleted file mode 100644 index d0219f3db73d0c260420b642f997663acdd36f87..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/AppTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.taosdata.jdbc; - -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -/** - * Unit test for simple App. - */ -public class AppTest { - /** - * Rigorous Test :-) - */ - @Test - public void shouldAnswerWithTrue() { - assertTrue(true); - } -} diff --git a/tests/examples/JDBC/calciteDemo/pom.xml b/tests/examples/JDBC/calciteDemo/pom.xml deleted file mode 100644 index a7d47837d9ba407f3959a00d522a9cd26b5622db..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/calciteDemo/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - com.taosdata.example.calcite - calciteDemo - 1.0-SNAPSHOT - - - - - org.slf4j - slf4j-simple - 1.7.25 - compile - - - - org.apache.calcite - calcite-core - 1.23.0 - - - org.apache.commons - commons-dbcp2 - 2.7.0 - - - org.apache.calcite.avatica - avatica-core - 1.17.0 - - - - - mysql - mysql-connector-java - 5.1.47 - - - - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.8 - - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java b/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java deleted file mode 100644 index 7e97956b7879c26056b9b0e27cce4786284ff96c..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.taosdata.example.calcite; - -import org.apache.calcite.adapter.jdbc.JdbcSchema; -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.schema.Schema; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.commons.dbcp2.BasicDataSource; - -import java.sql.*; -import java.util.Properties; - -public class CalciteDemo { - - private static String url_taos = "jdbc:TAOS://192.168.236.135:6030/test"; - private static String url_mysql = "jdbc:mysql://master:3306/test?useSSL=false&useUnicode=true&characterEncoding=UTF-8"; - - public static void main(String[] args) throws SqlParseException, ClassNotFoundException, SQLException { - Class.forName("org.apache.calcite.jdbc.Driver"); - Properties info = new Properties(); - info.setProperty("caseSensitive", "false"); - - Connection connection = DriverManager.getConnection("jdbc:calcite:", info); - CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); - - SchemaPlus rootSchema = calciteConnection.getRootSchema(); - - //这里hdb是在tdengine中创建的数据库名 - Schema schema = mysqlTest(rootSchema); -// Schema schema = tdengineTest(rootSchema); - - //创建新的schema自动映射到原来的hdb数据库 - rootSchema.add("test", schema); - - Statement stmt = calciteConnection.createStatement(); - //查询schema test中的表，表名是tdengine中的表 - ResultSet rs = stmt.executeQuery("select * from test.t"); - ResultSetMetaData metaData = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - System.out.println(metaData.getColumnLabel(i) + " : " + rs.getString(i)); - } - } - } - - - private static Schema tdengineTest(SchemaPlus rootSchema) throws ClassNotFoundException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(url_taos); - dataSource.setUsername("root"); - dataSource.setPassword("taosdata"); - - return JdbcSchema.create(rootSchema, "test", dataSource, "hdb", null); - } - - private static Schema mysqlTest(SchemaPlus rootSchema) throws ClassNotFoundException { - Class.forName("com.mysql.jdbc.Driver"); - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(url_mysql); - dataSource.setUsername("root"); - dataSource.setPassword("123456"); - - //Schema schema = JdbcSchema.create(rootSchema, "test", dataSource, "hdb", null); - return JdbcSchema.create(rootSchema, "test", dataSource, "test", null); - } -} diff --git a/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties b/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties deleted file mode 100644 index 1a77ec520cd3e727ac51acb4b8f647b7cf188d97..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties +++ /dev/null @@ -1,6 +0,0 @@ -log4j.rootLogger=info,stdout - -#console -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern= [%d{yyyy-MM-dd HH:mm:ss a}]:%p %l%m%n \ No newline at end of file diff --git a/tests/examples/JDBC/connectionPools/README-cn.md b/tests/examples/JDBC/connectionPools/README-cn.md index 761596dfc55a3e2c9f449ed34fd72ac96c277512..9b26df3c2eb2c23171a673643891a292af4c920c 100644 --- a/tests/examples/JDBC/connectionPools/README-cn.md +++ b/tests/examples/JDBC/connectionPools/README-cn.md @@ -1,23 +1,25 @@ 这个example中，我们适配了java常见的连接池: -* c3p0 -* dbcp +* HikariCP（默认） * druid -* HikariCP +* dbcp +* c3p0 ### 说明 ConnectionPoolDemo的程序逻辑： 1. 创建到host的connection连接池 2. 创建名称为pool_test的database，创建表超级weather，创建tableSize个子表 -3. 不断向所有子表进行插入。 +3. 总共插入totalNumber条数据。 ### 如何运行这个例子： + ```shell script -# mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args="-host localhost" +mvn clean package assembly:single +java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 ``` 使用mvn运行ConnectionPoolDemo的main方法，可以指定参数 ```shell script Usage: -mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args="" +java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar -host : hostname -poolType -poolSize @@ -26,8 +28,5 @@ mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.arg -sleep : 每次插入任务提交后的 ``` -### 如何停止程序： -ConnectionPoolDemo不会自己停止，会一直执行插入，需要手动Ctrl+C运行。 - ### 日志 使用log4j，将日志和错误分别输出到了debug.log和error.log中 \ No newline at end of file diff --git a/tests/examples/JDBC/connectionPools/pom.xml b/tests/examples/JDBC/connectionPools/pom.xml index 2793f0a83ddc88711796c133802c82979ae14be5..84467003f905c454bb285088e56bcd26465a3e5e 100644 --- a/tests/examples/JDBC/connectionPools/pom.xml +++ b/tests/examples/JDBC/connectionPools/pom.xml @@ -12,7 +12,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.11 + 2.0.18 @@ -44,13 +44,41 @@ c3p0 0.9.5.4 - log4j log4j 1.2.17 - + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + com.taosdata.example.ConnectionPoolDemo + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java deleted file mode 100644 index ed86acd6e9f8bfb8c862c1764e39f541d3f054eb..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.taosdata.demo.common; - -import org.apache.log4j.Logger; - -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Random; - -public class InsertTask implements Runnable { - private final Random random = new Random(System.currentTimeMillis()); - private static final Logger logger = Logger.getLogger(InsertTask.class); - - private final DataSource ds; - private final int batchSize; - private final String dbName; - private final int tableSize; - - public InsertTask(DataSource ds, String dbName, int tableSize, int batchSize) { - this.ds = ds; - this.dbName = dbName; - this.tableSize = tableSize; - this.batchSize = batchSize; - } - - @Override - public void run() { - Connection conn = null; - Statement stmt = null; - int affectedRows = 0; - - long start = System.currentTimeMillis(); - try { - conn = ds.getConnection(); - stmt = conn.createStatement(); - - for (int tb_index = 1; tb_index <= tableSize; tb_index++) { - StringBuilder sb = new StringBuilder(); - sb.append("insert into "); - sb.append(dbName); - sb.append(".t_"); - sb.append(tb_index); - sb.append("(ts, temperature, humidity) values "); - for (int i = 0; i < batchSize; i++) { - sb.append("("); - sb.append(start + i); - sb.append(", "); - sb.append(random.nextFloat() * 30); - sb.append(", "); - sb.append(random.nextInt(70)); - sb.append(") "); - } - logger.info("SQL >>> " + sb.toString()); - affectedRows += stmt.executeUpdate(sb.toString()); - } - } catch (SQLException e) { - e.printStackTrace(); - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - logger.info(">>> affectedRows:" + affectedRows + " TimeCost:" + (System.currentTimeMillis() - start) + " ms"); - } - } -} diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java deleted file mode 100644 index 87f1f4ad2cbba41a779f0247f2214ef2bf04a8ca..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.taosdata.demo.pool; - -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; - -import javax.sql.DataSource; - -public class HikariCpBuilder { - - public static DataSource getDataSource(String host, int poolSize) { - HikariConfig config = new HikariConfig(); - config.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); - config.setJdbcUrl("jdbc:TAOS://" + host + ":6030"); - config.setUsername("root"); - config.setPassword("taosdata"); - - config.setMaximumPoolSize(poolSize); - config.setMinimumIdle(poolSize); - HikariDataSource ds = new HikariDataSource(config); - return ds; - } -} diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java similarity index 57% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index 79c0aacea740dcb6fca9780c7f64872c537c3225..bd57d138b21034f45569ab3dcfc8e1ad5b39263d 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -1,10 +1,10 @@ -package com.taosdata.demo; +package com.taosdata.example; -import com.taosdata.demo.common.InsertTask; -import com.taosdata.demo.pool.C3p0Builder; -import com.taosdata.demo.pool.DbcpBuilder; -import com.taosdata.demo.pool.DruidPoolBuilder; -import com.taosdata.demo.pool.HikariCpBuilder; +import com.taosdata.example.common.InsertTask; +import com.taosdata.example.pool.C3p0Builder; +import com.taosdata.example.pool.DbcpBuilder; +import com.taosdata.example.pool.DruidPoolBuilder; +import com.taosdata.example.pool.HikariCpBuilder; import org.apache.log4j.Logger; import javax.sql.DataSource; @@ -20,44 +20,43 @@ public class ConnectionPoolDemo { private static Logger logger = Logger.getLogger(DruidPoolBuilder.class); private static final String dbName = "pool_test"; - private static int batchSize = 10; - private static int sleep = 1000; - private static int poolSize = 50; - private static int tableSize = 1000; - private static int threadCount = 50; private static String poolType = "hikari"; + private static long totalSize = 1_000_000l; + private static long tableSize = 1; + private static long batchSize = 1; + private static int poolSize = 50; + private static int threadCount = 50; + private static int sleep = 0; - public static void main(String[] args) throws InterruptedException { + public static void main(String[] args) { String host = null; for (int i = 0; i < args.length; i++) { if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { host = args[++i]; } - if ("-batchSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - batchSize = Integer.parseInt(args[++i]); - } - if ("-sleep".equalsIgnoreCase(args[i]) && i < args.length - 1) { - sleep = Integer.parseInt(args[++i]); + if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { + poolType = args[++i]; } - if ("-poolSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - poolSize = Integer.parseInt(args[++i]); + if ("-recordNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + totalSize = Long.parseLong(args[++i]); } - if ("-tableSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - tableSize = Integer.parseInt(args[++i]); + if ("-tableNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + tableSize = Long.parseLong(args[++i]); } - if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { - poolType = args[++i]; + if ("-batchNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + batchSize = Long.parseLong(args[++i]); } + } if (host == null) { - System.out.println("Usage: java -jar XXX.jar " + - "-host " + - "-batchSize " + - "-sleep " + - "-poolSize " + - "-tableSize " + - "-poolType "); + System.out.println("Usage: java -jar XXX.jar -host " + + "-poolType " + + "-recordNumber " + + "-tableNumber " + + "-batchNumber " + + "-sleep " + ); return; } @@ -79,15 +78,35 @@ public class ConnectionPoolDemo { } logger.info(">>>>>>>>>>>>>> connection pool Type: " + poolType); - init(dataSource); +// try { +// Connection connection = dataSource.getConnection(); +// Statement statement = connection.createStatement(); +// String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; +// int affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// statement.close(); +// connection.close(); +// } catch (SQLException e) { +// e.printStackTrace(); +// } + ExecutorService executor = Executors.newFixedThreadPool(threadCount); - while (true) { + for (long i = 0; i < totalSize / tableSize / batchSize; i++) { executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); - if (sleep > 0) - TimeUnit.MILLISECONDS.sleep(sleep); + // sleep few seconds + try { + if (sleep > 0) + TimeUnit.MILLISECONDS.sleep(sleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } } + executor.shutdown(); + } private static void init(DataSource dataSource) { diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java new file mode 100644 index 0000000000000000000000000000000000000000..da7c9a22b5b3e7f5d877a3a1489d55f439bff883 --- /dev/null +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java @@ -0,0 +1,46 @@ +package com.taosdata.example.common; + +import org.apache.log4j.Logger; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Random; + +public class InsertTask implements Runnable { + private final Random random = new Random(System.currentTimeMillis()); + private static final Logger logger = Logger.getLogger(InsertTask.class); + + private final DataSource ds; + private final String dbName; + private final long tableSize; + private final long batchSize; + + public InsertTask(DataSource ds, String dbName, long tableSize, long batchSize) { + this.ds = ds; + this.dbName = dbName; + this.tableSize = tableSize; + this.batchSize = batchSize; + } + + @Override + public void run() { + int affectedRows = 0; + long start = System.currentTimeMillis(); + try (Connection conn = ds.getConnection(); Statement stmt = conn.createStatement()) { + for (int tb_index = 1; tb_index <= tableSize; tb_index++) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(dbName).append(".t_").append(tb_index).append("(ts, temperature, humidity) values "); + for (int i = 0; i < batchSize; i++) { + sb.append("(").append(start + i).append(", ").append(random.nextFloat() * 30).append(", ").append(random.nextInt(70)).append(") "); + } + logger.info("SQL >>> " + sb.toString()); + affectedRows += stmt.executeUpdate(sb.toString()); + } + } catch (SQLException e) { + e.printStackTrace(); + } + logger.info(">>> affectedRows:" + affectedRows + " TimeCost:" + (System.currentTimeMillis() - start) + " ms"); + } +} diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java similarity index 89% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java index 587f417410f96f43be2ced5a4820cd49cdb99a17..235db0bb796ee507b2d93838c8e7bfa7de870fcd 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java @@ -1,7 +1,6 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import com.mchange.v2.c3p0.ComboPooledDataSource; -import org.apache.commons.dbcp.BasicDataSource; import javax.sql.DataSource; import java.beans.PropertyVetoException; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java similarity index 93% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java index 3c34a32532f595bf3134942094e96e952bd09dbb..3aa9e4ebcf2d07ba1e6a51a2dae216cf44e88486 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import org.apache.commons.dbcp.BasicDataSource; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java similarity index 74% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java index e5dc14c6a5ef69c2a7059d5d78b621e25ff3d799..500f0e9e97cc09c2c5e9990234f051d0bbc72044 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import com.alibaba.druid.pool.DruidDataSource; @@ -10,22 +10,18 @@ public class DruidPoolBuilder { final String url = "jdbc:TAOS://" + host + ":6030"; DruidDataSource dataSource = new DruidDataSource(); - dataSource.setUrl(url); + // jdbc properties dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setUrl(url); dataSource.setUsername("root"); dataSource.setPassword("taosdata"); - - //初始连接数，默认0 + // pool configurations dataSource.setInitialSize(poolSize); - //最大连接数，默认8 - dataSource.setMaxActive(poolSize); - //最小闲置数 dataSource.setMinIdle(poolSize); - //获取连接的最大等待时间，单位毫秒 - dataSource.setMaxWait(2000); - + dataSource.setMaxActive(poolSize); + dataSource.setMaxWait(30000); + dataSource.setValidationQuery("select server_status()"); return dataSource; } - } diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java new file mode 100644 index 0000000000000000000000000000000000000000..7e151de3e052495b3ce50ffe042732717fcc38f4 --- /dev/null +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java @@ -0,0 +1,28 @@ +package com.taosdata.example.pool; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import javax.sql.DataSource; + +public class HikariCpBuilder { + + public static DataSource getDataSource(String host, int poolSize) { + HikariConfig config = new HikariConfig(); + // jdbc properties + config.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + config.setJdbcUrl("jdbc:TAOS://" + host + ":6030"); + config.setUsername("root"); + config.setPassword("taosdata"); + // pool configurations + config.setMinimumIdle(poolSize); //minimum number of idle connection + config.setMaximumPoolSize(poolSize); //maximum number of connection in the pool + config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setMaxLifetime(0); // maximum life time for each connection + config.setIdleTimeout(0); // max idle time for recycle idle connection + config.setConnectionTestQuery("select server_status()"); //validation query + + HikariDataSource ds = new HikariDataSource(config); + return ds; + } +} diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml index 8535f3b797dcf13bd47d968f735ba5e7873fad51..a83d0a00e69bdcffff2db8b17c763959dc67365b 100644 --- a/tests/examples/JDBC/mybatisplus-demo/pom.xml +++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml @@ -38,19 +38,18 @@ h2 runtime + + com.alibaba + druid + 1.1.17 + com.taosdata.jdbc taos-jdbcdriver - 2.0.11 + 2.0.18 - - - mysql - mysql-connector-java - 5.1.47 - org.springframework.boot spring-boot-starter-web diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml index 96667f28b8d45d74541609f3d44176534c609f23..38180c6d75a620a63bcaab9ec350d97e65f9dd16 100644 --- a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml +++ b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml @@ -1,24 +1,14 @@ spring: datasource: - # driver-class-name: org.h2.Driver - # schema: classpath:db/schema-mysql.sql - # data: classpath:db/data-mysql.sql - # url: jdbc:h2:mem:test - # username: root - # password: test - - # driver-class-name: com.mysql.jdbc.Driver - # url: jdbc:mysql://master:3306/test?useSSL=false - # username: root - # password: 123456 - driver-class-name: com.taosdata.jdbc.TSDBDriver - url: jdbc:TAOS://localhost:6030/mp_test + url: jdbc:TAOS://localhost:6030/mp_test?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8 user: root password: taosdata - charset: UTF-8 - locale: en_US.UTF-8 - timezone: UTC-8 + + druid: + initial-size: 5 + min-idle: 5 + max-active: 5 mybatis-plus: configuration: diff --git a/tests/examples/JDBC/readme.md b/tests/examples/JDBC/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..9a017f4feab148cb7c3fd4132360c3075c6573cb --- /dev/null +++ b/tests/examples/JDBC/readme.md @@ -0,0 +1,13 @@ +# TDengine examples + +| No. | Name | Describe | +| :--: | :----------------: | ------------------------------------------------------------ | +| 1 | JDBCDemo | Example codes for JDBC-JNI, JDBC-RESTful, Subscribe | +| 2 | connectionPools | Example codes for HikariCP, Druid, dbcp, c3p0 connection pools | +| 3 | SpringJdbcTemplate | Example codes for spring jdbcTemplate | +| 4 | mybatisplus-demo | Example codes for mybatis | +| 5 | springbootdemo | Example codes for springboot | +| 6 | taosdemo | This is an internal tool for testing Our JDBC-JNI, JDBC-RESTful, RESTful interfaces | + + +more detail: https://www.taosdata.com/cn//documentation20/connector-java/ \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index 881ea0d6bf485891a29427853bd1bbf9b294c2d8..bd5f7efbc0321962a3f3f41a17bc9bde5dee9b27 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -8,7 +8,7 @@ 2.2.1.RELEASE - com.taosdata.jdbc + com.taosdata.example springbootdemo 0.0.1-SNAPSHOT springbootdemo @@ -63,7 +63,9 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.20 + + @@ -71,8 +73,6 @@ druid-spring-boot-starter 1.1.17 - - diff --git a/tests/examples/JDBC/springbootdemo/readme.md b/tests/examples/JDBC/springbootdemo/readme.md index ac3bb44ef95c251e25c664cd50414494e19f6820..67a28947d2dfb8fc069bf94fd139a7006d35a22b 100644 --- a/tests/examples/JDBC/springbootdemo/readme.md +++ b/tests/examples/JDBC/springbootdemo/readme.md @@ -47,7 +47,7 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug * 插入单条记录 ```xml - + insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) ``` @@ -67,9 +67,9 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug - + - + diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java similarity index 75% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java index 69cd3e0ced2888575d890ffea36407455c4bea7a..8f30c299466ca1f83bc689928b08e8001a87908d 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java @@ -1,15 +1,13 @@ -package com.taosdata.jdbc.springbootdemo; +package com.taosdata.example.springbootdemo; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@MapperScan(basePackages = {"com.taosdata.jdbc.springbootdemo.dao"}) +@MapperScan(basePackages = {"com.taosdata.example.springbootdemo.dao"}) @SpringBootApplication public class SpringbootdemoApplication { - public static void main(String[] args) { SpringApplication.run(SpringbootdemoApplication.class, args); } - -} +} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java similarity index 60% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index 56a58fbb4d9a783a34989eae00369ff7b307acaf..c153e27701403b672709a3585603b8630796d178 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -1,11 +1,12 @@ -package com.taosdata.jdbc.springbootdemo.controller; +package com.taosdata.example.springbootdemo.controller; -import com.taosdata.jdbc.springbootdemo.domain.Weather; -import com.taosdata.jdbc.springbootdemo.service.WeatherService; +import com.taosdata.example.springbootdemo.domain.Weather; +import com.taosdata.example.springbootdemo.service.WeatherService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.util.List; +import java.util.Map; @RequestMapping("/weather") @RestController @@ -20,7 +21,7 @@ public class WeatherController { * @return */ @GetMapping("/init") - public boolean init() { + public int init() { return weatherService.init(); } @@ -44,21 +45,23 @@ public class WeatherController { * @return */ @PostMapping("/{temperature}/{humidity}") - public int saveWeather(@PathVariable int temperature, @PathVariable float humidity) { - + public int saveWeather(@PathVariable float temperature, @PathVariable int humidity) { return weatherService.save(temperature, humidity); } - /** - * upload multi weather info - * - * @param weatherList - * @return - */ - @PostMapping("/batch") - public int batchSaveWeather(@RequestBody List weatherList) { + @GetMapping("/count") + public int count() { + return weatherService.count(); + } + + @GetMapping("/subTables") + public List getSubTables() { + return weatherService.getSubTables(); + } - return weatherService.save(weatherList); + @GetMapping("/avg") + public List avg() { + return weatherService.avg(); } } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..ad6733558a9d548be196cf8c9c0c63dc96227b39 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -0,0 +1,29 @@ +package com.taosdata.example.springbootdemo.dao; + +import com.taosdata.example.springbootdemo.domain.Weather; +import org.apache.ibatis.annotations.Param; + +import java.util.List; +import java.util.Map; + +public interface WeatherMapper { + + void dropDB(); + + void createDB(); + + void createSuperTable(); + + void createTable(Weather weather); + + List select(@Param("limit") Long limit, @Param("offset") Long offset); + + int insert(Weather weather); + + int count(); + + List getSubTables(); + + List avg(); + +} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml new file mode 100644 index 0000000000000000000000000000000000000000..2d3e0540650f35c1018992795ac33fb6cb7c4837 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + drop database if exists test + + + + create database if not exists test + + + + create table if not exists test.weather(ts timestamp, temperature float, humidity float) tags(location nchar(64), groupId int) + + + + create table if not exists test.t#{groupId} using test.weather tags(#{location}, #{groupId}) + + + + + + insert into test.t#{groupId} (ts, temperature, humidity) values (#{ts}, ${temperature}, ${humidity}) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java new file mode 100644 index 0000000000000000000000000000000000000000..255b2cdca920a5568338e823d0744f2e9c4db7d3 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java @@ -0,0 +1,64 @@ +package com.taosdata.example.springbootdemo.domain; + +import com.fasterxml.jackson.annotation.JsonFormat; + +import java.sql.Timestamp; + +public class Weather { + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS", timezone = "GMT+8") + private Timestamp ts; + private float temperature; + private float humidity; + private String location; + private int groupId; + + public Weather() { + } + + public Weather(Timestamp ts, float temperature, float humidity) { + this.ts = ts; + this.temperature = temperature; + this.humidity = humidity; + } + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public float getTemperature() { + return temperature; + } + + public void setTemperature(float temperature) { + this.temperature = temperature; + } + + public float getHumidity() { + return humidity; + } + + public void setHumidity(float humidity) { + this.humidity = humidity; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public int getGroupId() { + return groupId; + } + + public void setGroupId(int groupId) { + this.groupId = groupId; + } +} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java new file mode 100644 index 0000000000000000000000000000000000000000..0aef828e1cf19f9c612f9c8d0433ce1a361c7441 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -0,0 +1,61 @@ +package com.taosdata.example.springbootdemo.service; + +import com.taosdata.example.springbootdemo.dao.WeatherMapper; +import com.taosdata.example.springbootdemo.domain.Weather; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; +import java.util.Random; + +@Service +public class WeatherService { + + @Autowired + private WeatherMapper weatherMapper; + private Random random = new Random(System.currentTimeMillis()); + private String[] locations = {"北京", "上海", "广州", "深圳", "天津"}; + + public int init() { + weatherMapper.dropDB(); + weatherMapper.createDB(); + weatherMapper.createSuperTable(); + long ts = System.currentTimeMillis(); + long thirtySec = 1000 * 30; + int count = 0; + for (int i = 0; i < 20; i++) { + Weather weather = new Weather(new Timestamp(ts + (thirtySec * i)), 30 * random.nextFloat(), random.nextInt(100)); + weather.setLocation(locations[random.nextInt(locations.length)]); + weather.setGroupId(i % locations.length); + weatherMapper.createTable(weather); + count += weatherMapper.insert(weather); + } + return count; + } + + public List query(Long limit, Long offset) { + return weatherMapper.select(limit, offset); + } + + public int save(float temperature, int humidity) { + Weather weather = new Weather(); + weather.setTemperature(temperature); + weather.setHumidity(humidity); + + return weatherMapper.insert(weather); + } + + public int count() { + return weatherMapper.count(); + } + + public List getSubTables() { + return weatherMapper.getSubTables(); + } + + public List avg() { + return weatherMapper.avg(); + } +} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java deleted file mode 100644 index 844ac21bb8855de06edcc586deb713bcad686385..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.controller; - - -import com.taosdata.jdbc.springbootdemo.domain.Rainfall; -import com.taosdata.jdbc.springbootdemo.service.RainStationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -@RestController -@RequestMapping("/rainstation") -public class RainStationController { - - @Autowired - private RainStationService service; - - @GetMapping("/init") - public boolean init() { - service.init(); - service.createTable(); - return true; - } - - @PostMapping("/insert") - public int insert(@RequestBody Rainfall rainfall){ - return service.insert(rainfall); - } - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java deleted file mode 100644 index a9266acb3028c453062ce6e56ccade53c405f742..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import java.util.Map; - -public interface DatabaseMapper { - - int createDatabase(String dbname); - - int dropDatabase(String dbname); - - int creatDatabaseWithParameters(Map map); - - int useDatabase(String dbname); - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml deleted file mode 100644 index 329f75b582a276bed522e671d0c9f868b837c172..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - create database if not exists ${dbname} - - - - DROP database if exists ${dbname} - - - - - CREATE database if not EXISTS ${dbname} - - KEEP ${keep} - - - DAYS ${days} - - - REPLICA ${replica} - - - cache ${cache} - - - blocks ${blocks} - - - minrows ${minrows} - - - maxrows ${maxrows} - - - - - use ${dbname} - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java deleted file mode 100644 index f0efbf40ba477d9b8f410e0806be38701d274394..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import java.util.Map; - -public interface RainfallMapper { - - - int save(Map map); -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml deleted file mode 100644 index 319b4f397434f1b7623c31250c8ba798c0499d86..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - INSERT INTO ${table} using ${dbname}.${stable} tags(#{values.station_code}, #{values.station_name}) (ts, name, code, rainfall) values (#{values.ts}, #{values.name}, #{values.code}, #{values.rainfall}) - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java deleted file mode 100644 index 7601bf974c7c727b2a46171b145bdcec662dc258..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import com.taosdata.jdbc.springbootdemo.domain.TableMetadata; - -public interface TableMapper { - - boolean createSTable(TableMetadata tableMetadata); -} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml deleted file mode 100644 index 5a272eadb4abd7637735a834ec445c822073486f..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - create table if not exists ${dbname}.${tablename} - - ${field.name} ${field.type} - - TAGS - - ${tag.name} ${tag.type} - - - - - drop ${tablename} - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java deleted file mode 100644 index 1e3db1f49106606c412851c0a74ad382adea68fb..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import com.taosdata.jdbc.springbootdemo.domain.Weather; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -public interface WeatherMapper { - - int insert(Weather weather); - - int batchInsert(List weatherList); - - List select(@Param("limit") Long limit, @Param("offset")Long offset); - - void createDB(); - - void createTable(); -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml deleted file mode 100644 index e894f9a6583d271d8ce526e9afe79528f0fd5490..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - create database if not exists test; - - - - create table if not exists test.weather(ts timestamp, temperature int, humidity float); - - - - ts, temperature, humidity - - - - - - insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) - - - - insert into test.weather (ts, temperature, humidity) values - - (now + #{index}a, #{weather.temperature}, #{weather.humidity}) - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java deleted file mode 100644 index 619b5a303d715132d6a428770a27d069a9b9dcaa..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -public class FieldMetadata { - - private String name; - private String type; - - public FieldMetadata(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java deleted file mode 100644 index 93e199d7e66582e7f8d771994b358a2c58f5dae2..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import com.fasterxml.jackson.annotation.JsonFormat; - -import java.sql.Timestamp; - -public class Rainfall { - - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8") - private Timestamp ts; - private String name; - private String code; - private float rainfall; - private String station_code; - private String station_name; - - public Timestamp getTs() { - return ts; - } - - public void setTs(Timestamp ts) { - this.ts = ts; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - - public float getRainfall() { - return rainfall; - } - - public void setRainfall(float rainfall) { - this.rainfall = rainfall; - } - - public String getStation_code() { - return station_code; - } - - public void setStation_code(String station_code) { - this.station_code = station_code; - } - - public String getStation_name() { - return station_name; - } - - public void setStation_name(String station_name) { - this.station_name = station_name; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java deleted file mode 100644 index 74bb434f08484af6c244dd884570c974ab93e634..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import java.util.List; - -public class TableMetadata { - - private String dbname; - private String tablename; - private List fields; - private List tags; - - public String getDbname() { - return dbname; - } - - public void setDbname(String dbname) { - this.dbname = dbname; - } - - public String getTablename() { - return tablename; - } - - public void setTablename(String tablename) { - this.tablename = tablename; - } - - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } - - public List getTags() { - return tags; - } - - public void setTags(List tags) { - this.tags = tags; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java deleted file mode 100644 index 755ecc0075506c4ba2c99eafa47593a69a50bd84..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -public class TagMetadata { - private String name; - private String type; - - public TagMetadata(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java deleted file mode 100644 index cd7de447eade4a126c86c38c703d6a4de444c557..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import com.fasterxml.jackson.annotation.JsonFormat; - -import java.sql.Timestamp; - -public class Weather { - - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8") - private Timestamp ts; - - private int temperature; - - private float humidity; - - public Timestamp getTs() { - return ts; - } - - public void setTs(Timestamp ts) { - this.ts = ts; - } - - public int getTemperature() { - return temperature; - } - - public void setTemperature(int temperature) { - this.temperature = temperature; - } - - public float getHumidity() { - return humidity; - } - - public void setHumidity(float humidity) { - this.humidity = humidity; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java deleted file mode 100644 index 3ea63c1760d4aef9e11a076eaff0c19df3a3a73c..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.service; - -import com.taosdata.jdbc.springbootdemo.dao.DatabaseMapper; -import com.taosdata.jdbc.springbootdemo.dao.RainfallMapper; -import com.taosdata.jdbc.springbootdemo.dao.TableMapper; -import com.taosdata.jdbc.springbootdemo.domain.FieldMetadata; -import com.taosdata.jdbc.springbootdemo.domain.Rainfall; -import com.taosdata.jdbc.springbootdemo.domain.TableMetadata; -import com.taosdata.jdbc.springbootdemo.domain.TagMetadata; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Service -public class RainStationService { - - @Autowired - private DatabaseMapper databaseMapper; - @Autowired - private TableMapper tableMapper; - @Autowired - private RainfallMapper rainfallMapper; - - public boolean init() { - databaseMapper.dropDatabase("rainstation"); - - Map map = new HashMap<>(); - map.put("dbname", "rainstation"); - map.put("keep", "36500"); - map.put("days", "30"); - map.put("blocks", "4"); - databaseMapper.creatDatabaseWithParameters(map); - - databaseMapper.useDatabase("rainstation"); - return true; - } - - public boolean createTable() { - TableMetadata tableMetadata = new TableMetadata(); - tableMetadata.setDbname("rainstation"); - tableMetadata.setTablename("monitoring"); - - List fields = new ArrayList<>(); - fields.add(new FieldMetadata("ts", "timestamp")); - fields.add(new FieldMetadata("name", "NCHAR(10)")); - fields.add(new FieldMetadata("code", " BINARY(8)")); - fields.add(new FieldMetadata("rainfall", "float")); - tableMetadata.setFields(fields); - - List tags = new ArrayList<>(); - tags.add(new TagMetadata("station_code", "BINARY(8)")); - tags.add(new TagMetadata("station_name", "NCHAR(10)")); - tableMetadata.setTags(tags); - - tableMapper.createSTable(tableMetadata); - return true; - } - - - public int insert(Rainfall rainfall) { - Map map = new HashMap<>(); - map.put("dbname", "rainstation"); - map.put("table", "S_53646"); - map.put("stable", "monitoring"); - map.put("values", rainfall); - return rainfallMapper.save(map); - } -} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java deleted file mode 100644 index b950a9a4e42e306320415c7be556ba78c2f25a27..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.service; - -import com.taosdata.jdbc.springbootdemo.dao.WeatherMapper; -import com.taosdata.jdbc.springbootdemo.domain.Weather; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class WeatherService { - - @Autowired - private WeatherMapper weatherMapper; - - public boolean init() { - weatherMapper.createDB(); - weatherMapper.createTable(); - return true; - } - - public List query(Long limit, Long offset) { - return weatherMapper.select(limit, offset); - } - - public int save(int temperature, float humidity) { - Weather weather = new Weather(); - weather.setTemperature(temperature); - weather.setHumidity(humidity); - - return weatherMapper.insert(weather); - } - - public int save(List weatherList) { - return weatherMapper.batchInsert(weatherList); - } - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 683884fcdd852fd1a72c685b40c6a8517ed92ff4..4d7e64d10576388827502a459df9e68da2721dbb 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,24 +1,20 @@ -# datasource config -spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -spring.datasource.url=jdbc:TAOS://localhost:6030/log +# datasource config - JDBC-JNI +#spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver +#spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.username=root +#spring.datasource.password=taosdata + +# datasource config - JDBC-RESTful +spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver +spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 spring.datasource.druid.max-active=5 -# max wait time for get connection, ms -spring.datasource.druid.max-wait=60000 - +spring.datasource.druid.max-wait=30000 spring.datasource.druid.validation-query=select server_status(); -spring.datasource.druid.validation-query-timeout=5000 -spring.datasource.druid.test-on-borrow=false -spring.datasource.druid.test-on-return=false -spring.datasource.druid.test-while-idle=true -spring.datasource.druid.time-between-eviction-runs-millis=60000 -spring.datasource.druid.min-evictable-idle-time-millis=600000 -spring.datasource.druid.max-evictable-idle-time-millis=900000 - #mybatis mybatis.mapper-locations=classpath:mapper/*.xml diff --git a/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java b/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java deleted file mode 100644 index 23a7420dab24a15d9d24341839ba58caa9acb4b9..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.taosdata.jdbc.springbootdemo; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class SpringbootdemoApplicationTests { - - @Test - void contextLoads() { - } - -} diff --git a/tests/examples/JDBC/taosdemo/.gitignore b/tests/examples/JDBC/taosdemo/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..549e00a2a96fa9d7c5dbc9859664a78d980158c2 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/tests/examples/JDBC/taosdemo/.mvn/wrapper/MavenWrapperDownloader.java b/tests/examples/JDBC/taosdemo/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 0000000000000000000000000000000000000000..a45eb6ba269cd38f8965cef786729790945d9537 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,118 @@ +/* + * Copyright 2007-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + private static final String WRAPPER_VERSION = "0.5.6"; + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if (mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if (mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if (!outputFile.getParentFile().exists()) { + if (!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { + String username = System.getenv("MVNW_USERNAME"); + char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password); + } + }); + } + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.jar b/tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..2cc7d4a55c0cd0092912bf49ae38b3a9e3fd0054 Binary files /dev/null and b/tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.jar differ diff --git a/tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.properties b/tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000000000000000000000000000000000000..642d572ce90e5085986bdd9c9204b9404f028084 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/tests/examples/JDBC/taosdemo/mvnw b/tests/examples/JDBC/taosdemo/mvnw new file mode 100755 index 0000000000000000000000000000000000000000..3c8a5537314954d53ec2fb774b34fe5d5a5f253a --- /dev/null +++ b/tests/examples/JDBC/taosdemo/mvnw @@ -0,0 +1,322 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ]; then + + if [ -f /etc/mavenrc ]; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ]; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false +darwin=false +mingw=false +case "$(uname)" in +CYGWIN*) cygwin=true ;; +MINGW*) mingw=true ;; +Darwin*) + darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="$(/usr/libexec/java_home)" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ]; then + if [ -r /etc/gentoo-release ]; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +if [ -z "$M2_HOME" ]; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ]; do + ls=$(ls -ld "$PRG") + link=$(expr "$ls" : '.*-> \(.*\)$') + if expr "$link" : '/.*' >/dev/null; then + PRG="$link" + else + PRG="$(dirname "$PRG")/$link" + fi + done + + saveddir=$(pwd) + + M2_HOME=$(dirname "$PRG")/.. + + # make it fully qualified + M2_HOME=$(cd "$M2_HOME" && pwd) + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=$(cygpath --unix "$M2_HOME") + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw; then + [ -n "$M2_HOME" ] && + M2_HOME="$( ( + cd "$M2_HOME" + pwd + ))" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="$( ( + cd "$JAVA_HOME" + pwd + ))" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr \"$javaExecutable\" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! $(expr "$readLink" : '\([^ ]*\)') = "no" ]; then + if $darwin; then + javaHome="$(dirname \"$javaExecutable\")" + javaExecutable="$(cd \"$javaHome\" && pwd -P)/javac" + else + javaExecutable="$(readlink -f \"$javaExecutable\")" + fi + javaHome="$(dirname \"$javaExecutable\")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ]; then + if [ -n "$JAVA_HOME" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(which java)" + fi +fi + +if [ ! -x "$JAVACMD" ]; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ]; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ]; then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ]; do + if [ -d "$wdir"/.mvn ]; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$( + cd "$wdir/.." + pwd + ) + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' <"$1")" + fi +} + +BASE_DIR=$(find_maven_basedir "$(pwd)") +if [ -z "$BASE_DIR" ]; then + exit 1 +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in wrapperUrl) + jarUrl="$value" + break + ;; + esac + done <"$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget >/dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl >/dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=$(cygpath --path --windows "$M2_HOME") + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/tests/examples/JDBC/taosdemo/mvnw.cmd b/tests/examples/JDBC/taosdemo/mvnw.cmd new file mode 100644 index 0000000000000000000000000000000000000000..c8d43372c986d97911cdc21bd87e0cbe3d83bdda --- /dev/null +++ b/tests/examples/JDBC/taosdemo/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/tests/examples/JDBC/taosdemo/pom.xml b/tests/examples/JDBC/taosdemo/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..22c2f3b63e82a3a2cdcc3093c3c43b98ab534a4b --- /dev/null +++ b/tests/examples/JDBC/taosdemo/pom.xml @@ -0,0 +1,168 @@ + + + 4.0.0 + com.taosdata + taosdemo + 2.0 + taosdemo + jar + Demo project for TDengine + + + 5.3.2 + + + + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-core + ${spring.version} + + + org.springframework + spring-beans + ${spring.version} + + + org.springframework + spring-expression + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-aspects + ${spring.version} + + + org.springframework + spring-test + ${spring.version} + test + + + org.springframework + spring-jdbc + ${spring.version} + + + + + com.zaxxer + HikariCP + 3.4.5 + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.20 + + + + + + com.alibaba + fastjson + 1.2.75 + + + + mysql + mysql-connector-java + 5.1.47 + test + + + + log4j + log4j + 1.2.17 + + + + junit + junit + 4.12 + test + + + + org.projectlombok + lombok + 1.18.16 + provided + + + + + + + src/main/resources + + **/*.properties + **/*.xml + **/*.jar + + true + + + src/main/java + + **/*.properties + **/*.xml + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + + com.taosdata.taosdemo.TaosDemoApplication + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + + + diff --git a/tests/examples/JDBC/taosdemo/readme.md b/tests/examples/JDBC/taosdemo/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..451fa2960adb98e2deb8499732aefde11f4810a1 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/readme.md @@ -0,0 +1,13 @@ +``` +cd tests/examples/JDBC/taosdemo +mvn clean package -Dmaven.test.skip=true +# 先建表，再插入的 +java -jar target/taosdemo-2.0-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 +# 不建表，直接插入的 +java -jar target/taosdemo-2.0-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 +``` + +需求： +1. 可以读lowa的配置文件 +2. 支持JDBC-JNI和JDBC-restful +3. 读取配置文件，持续执行查询 \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java new file mode 100644 index 0000000000000000000000000000000000000000..c361df82b0aebb0d804b1a0982a0c1cf44ef5953 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -0,0 +1,127 @@ +package com.taosdata.taosdemo; + +import com.taosdata.taosdemo.components.DataSourceFactory; +import com.taosdata.taosdemo.components.JdbcTaosdemoConfig; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.service.DatabaseService; +import com.taosdata.taosdemo.service.SqlExecuteTask; +import com.taosdata.taosdemo.service.SubTableService; +import com.taosdata.taosdemo.service.SuperTableService; +import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator; +import org.apache.log4j.Logger; + +import javax.sql.DataSource; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class TaosDemoApplication { + + private static final Logger logger = Logger.getLogger(TaosDemoApplication.class); + + public static void main(String[] args) throws IOException { + // 读配置参数 + JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); + boolean isHelp = Arrays.asList(args).contains("--help"); + if (isHelp || config.host == null || config.host.isEmpty()) { + JdbcTaosdemoConfig.printHelp(); + System.exit(0); + } + // 初始化 + final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); + if (config.executeSql != null && !config.executeSql.isEmpty() && !config.executeSql.replaceAll("\\s", "").isEmpty()) { + Thread task = new Thread(new SqlExecuteTask(dataSource, config.executeSql)); + task.start(); + try { + task.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return; + } + + final DatabaseService databaseService = new DatabaseService(dataSource); + final SuperTableService superTableService = new SuperTableService(dataSource); + final SubTableService subTableService = new SubTableService(dataSource); + + // 创建数据库 + long start = System.currentTimeMillis(); + Map databaseParam = new HashMap<>(); + databaseParam.put("database", config.database); + databaseParam.put("keep", Integer.toString(config.keep)); + databaseParam.put("days", Integer.toString(config.days)); + databaseParam.put("replica", Integer.toString(config.replica)); + //TODO: other database parameters + databaseService.createDatabase(databaseParam); + databaseService.useDatabase(config.database); + long end = System.currentTimeMillis(); + logger.info(">>> create database time cost : " + (end - start) + " ms."); + /**********************************************************************************/ + // 构造超级表的meta + SuperTableMeta superTableMeta; + // create super table + if (config.superTableSQL != null) { + // use a sql to create super table + superTableMeta = SuperTableMetaGenerator.generate(config.superTableSQL); + if (config.database != null && !config.database.isEmpty()) + superTableMeta.setDatabase(config.database); + } else if (config.numOfFields == 0) { + String sql = "create table " + config.database + "." + config.superTable + " (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; + superTableMeta = SuperTableMetaGenerator.generate(sql); + } else { + // create super table with specified field size and tag size + superTableMeta = SuperTableMetaGenerator.generate(config.database, config.superTable, config.numOfFields, config.prefixOfFields, config.numOfTags, config.prefixOfTags); + } + /**********************************************************************************/ + // 建表 + start = System.currentTimeMillis(); + if (config.doCreateTable) { + superTableService.drop(superTableMeta.getDatabase(), superTableMeta.getName()); + superTableService.create(superTableMeta); + if (!config.autoCreateTable) { + // 批量建子表 + subTableService.createSubTable(superTableMeta, config.numOfTables, config.prefixOfTable, config.numOfThreadsForCreate); + } + } + end = System.currentTimeMillis(); + logger.info(">>> create table time cost : " + (end - start) + " ms."); + /**********************************************************************************/ + // 插入 + long tableSize = config.numOfTables; + int threadSize = config.numOfThreadsForInsert; + long startTime = getProperStartTime(config.startTime, config.keep); + + if (tableSize < threadSize) + threadSize = (int) tableSize; + long gap = (long) Math.ceil((0.0d + tableSize) / threadSize); + + start = System.currentTimeMillis(); + // multi threads to insert + int affectedRows = subTableService.insertMultiThreads(superTableMeta, threadSize, tableSize, startTime, gap, config); + end = System.currentTimeMillis(); + logger.info("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms"); + /**********************************************************************************/ + // 查询 + + + /**********************************************************************************/ + // 删除表 + if (config.dropTable) { + superTableService.drop(config.database, config.superTable); + } + System.exit(0); + } + + private static long getProperStartTime(long startTime, int keep) { + Instant now = Instant.now(); + long earliest = now.minus(Duration.ofDays(keep - 1)).toEpochMilli(); + if (startTime == 0 || startTime < earliest) { + startTime = earliest; + } + return startTime; + } + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..a7d08e96ea373c4773e872bcaf9b3a7b98d5bf9a --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java @@ -0,0 +1,62 @@ +package com.taosdata.taosdemo.components; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; + +import javax.sql.DataSource; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +@Component +public class DataSourceFactory { + + private static DataSource instance; + + public static DataSource getInstance(String host, int port, String user, String password) throws IOException { + if (instance == null) { + synchronized (DataSourceFactory.class) { + if (instance == null) { + InputStream is = DataSourceFactory.class.getClassLoader().getResourceAsStream("application.properties"); + Properties properties = new Properties(); + properties.load(is); + + HikariConfig config = new HikariConfig(); + if (properties.containsKey("jdbc.driver")) { +// String driverName = properties.getProperty("jdbc.driver"); +// System.out.println(">>> load driver : " + driverName); +// try { +// Class.forName(driverName); +// } catch (ClassNotFoundException e) { +// e.printStackTrace(); +// } + config.setDriverClassName(properties.getProperty("jdbc.driver")); + } else { + config.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + } + if ("com.taosdata.jdbc.rs.RestfulDriver".equalsIgnoreCase(properties.getProperty("jdbc.driver"))) + config.setJdbcUrl("jdbc:TAOS-RS://" + host + ":6041/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + else + config.setJdbcUrl("jdbc:TAOS://" + host + ":" + port + "/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + config.setUsername(user); + config.setPassword(password); + // maximum-pool-size + if (properties.containsKey("hikari.maximum-pool-size")) + config.setMaximumPoolSize(Integer.parseInt(properties.getProperty("hikari.maximum-pool-size"))); + else + config.setMaximumPoolSize(500); + // minimum-idle + if (properties.containsKey("hikari.minimum-idle")) + config.setMinimumIdle(Integer.parseInt(properties.getProperty("hikari.minimum-idle"))); + else + config.setMinimumIdle(100); + config.setMaxLifetime(0); + instance = new HikariDataSource(config); + } + } + } + return instance; + } + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..974a2755a5a029d3a5fc681bc8c59b0aca1a7ca4 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java @@ -0,0 +1,221 @@ +package com.taosdata.taosdemo.components; + +import com.taosdata.taosdemo.utils.TimeStampUtil; + +public final class JdbcTaosdemoConfig { + // instance + public String host; //host + public int port = 6030; //port + public String user = "root"; //user + public String password = "taosdata"; //password + // database + public String database = "jdbcdb"; //database + public int keep = 3650; //keep + public int days = 30; //days + public int replica = 1; //replica + public int blocks = 16; + public int cache = 8; + public String precision = "ms"; + + //super table + public boolean doCreateTable = true; + public String superTable = "weather"; //super table name + public String prefixOfFields = "col"; + public int numOfFields; + public String prefixOfTags = "tag"; + public int numOfTags; + public String superTableSQL; + //sub table + public String prefixOfTable = "t"; + // insert task + public boolean autoCreateTable; + public long numOfTables = 10; + public long numOfRowsPerTable = 10; + public int numOfTablesPerSQL = 1; + public int numOfValuesPerSQL = 1; + public int numOfThreadsForCreate = 1; + public int numOfThreadsForInsert = 1; + public long startTime; + public long timeGap = 1; + public int frequency; + public int order; + public int rate = 10; + public long range = 1000l; + // select task + public String executeSql; + // drop task + public boolean dropTable = false; + + public static void printHelp() { + System.out.println("Usage: java -jar jdbc-taosdemo-2.0.jar [OPTION...]"); + // instance + System.out.println("-host The host to connect to TDengine which you must specify"); + System.out.println("-port The TCP/IP port number to use for the connection. Default is 6030"); + System.out.println("-user The TDengine user name to use when connecting to the server. Default is 'root'"); + System.out.println("-password The password to use when connecting to the server.Default is 'taosdata'"); + // database + System.out.println("-database Destination database. Default is 'jdbcdb'"); + System.out.println("-keep database keep parameter. Default is 3650"); + System.out.println("-days database days parameter. Default is 30"); + System.out.println("-replica database replica parameter. Default 1, min: 1, max: 3"); + System.out.println("-blocks database blocks parameter. Default is 16"); + System.out.println("-cache database cache parameter. Default is 8"); + System.out.println("-precision database precision parameter. Default is ms"); + + // super table + System.out.println("-doCreateTable do create super table and sub table, true or false, Default true"); + System.out.println("-superTable super table name. Default 'weather'"); + System.out.println("-prefixOfFields The prefix of field in super table. Default is 'col'"); + System.out.println("-numOfFields The number of field in super table. Default is (ts timestamp, temperature float, humidity int)."); + System.out.println("-prefixOfTags The prefix of tag in super table. Default is 'tag'"); + System.out.println("-numOfTags The number of tag in super table. Default is (location nchar(64), groupId int)."); + System.out.println("-superTableSQL specify a sql statement for the super table.\n" + + " Default is 'create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int). \n" + + " if you use this parameter, the numOfFields and numOfTags will be invalid'"); + // sub table + System.out.println("-prefixOfTable The prefix of sub tables. Default is 't'"); + System.out.println("-numOfTables The number of tables. Default is 1"); + System.out.println("-numOfThreadsForCreate The number of thread during create sub table. Default is 1"); + // insert task + System.out.println("-autoCreateTable Use auto Create sub tables SQL. Default is false"); + System.out.println("-numOfRowsPerTable The number of records per table. Default is 1"); + System.out.println("-numOfThreadsForInsert The number of threads during insert row. Default is 1"); + System.out.println("-numOfTablesPerSQL The number of table per SQL. Default is 1"); + System.out.println("-numOfValuesPerSQL The number of value per SQL. Default is 1"); + System.out.println("-startTime start time for insert task, The format is \"yyyy-MM-dd HH:mm:ss.SSS\"."); + System.out.println("-timeGap the number of time gap. Default is 1000 ms"); + System.out.println("-frequency the number of records per second inserted into one table. default is 0, do not control frequency"); + System.out.println("-order Insert mode--0: In order, 1: Out of order. Default is in order"); + System.out.println("-rate The proportion of data out of order. effective only if order is 1. min 0, max 100, default is 10"); + System.out.println("-range The range of data out of order. effective only if order is 1. default is 1000 ms"); + // query task + System.out.println("-executeSql execute a specific sql."); + // drop task + System.out.println("-dropTable Drop data before quit. Default is false"); + System.out.println("--help Give this help list"); + } + + /** + * parse args from command line + * + * @param args command line args + * @return JdbcTaosdemoConfig + */ + public JdbcTaosdemoConfig(String[] args) { + for (int i = 0; i < args.length; i++) { + // instance + if ("-host".equals(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-port".equals(args[i]) && i < args.length - 1) { + port = Integer.parseInt(args[++i]); + } + if ("-user".equals(args[i]) && i < args.length - 1) { + user = args[++i]; + } + if ("-password".equals(args[i]) && i < args.length - 1) { + password = args[++i]; + } + // database + if ("-database".equals(args[i]) && i < args.length - 1) { + database = args[++i]; + } + if ("-keep".equals(args[i]) && i < args.length - 1) { + keep = Integer.parseInt(args[++i]); + } + if ("-days".equals(args[i]) && i < args.length - 1) { + days = Integer.parseInt(args[++i]); + } + if ("-replica".equals(args[i]) && i < args.length - 1) { + replica = Integer.parseInt(args[++i]); + } + if ("-blocks".equals(args[i]) && i < args.length - 1) { + blocks = Integer.parseInt(args[++i]); + } + if ("-cache".equals(args[i]) && i < args.length - 1) { + cache = Integer.parseInt(args[++i]); + } + if ("-precision".equals(args[i]) && i < args.length - 1) { + precision = args[++i]; + } + // super table + if ("-doCreateTable".equals(args[i]) && i < args.length - 1) { + doCreateTable = Boolean.parseBoolean(args[++i]); + } + if ("-superTable".equals(args[i]) && i < args.length - 1) { + superTable = args[++i]; + } + if ("-prefixOfFields".equals(args[i]) && i < args.length - 1) { + prefixOfFields = args[++i]; + } + if ("-numOfFields".equals(args[i]) && i < args.length - 1) { + numOfFields = Integer.parseInt(args[++i]); + } + if ("-prefixOfTags".equals(args[i]) && i < args.length - 1) { + prefixOfTags = args[++i]; + } + if ("-numOfTags".equals(args[i]) && i < args.length - 1) { + numOfTags = Integer.parseInt(args[++i]); + } + if ("-superTableSQL".equals(args[i]) && i < args.length - 1) { + superTableSQL = args[++i]; + } + // sub table + if ("-prefixOfTable".equals(args[i]) && i < args.length - 1) { + prefixOfTable = args[++i]; + } + if ("-numOfTables".equals(args[i]) && i < args.length - 1) { + numOfTables = Long.parseLong(args[++i]); + } + if ("-autoCreateTable".equals(args[i]) && i < args.length - 1) { + autoCreateTable = Boolean.parseBoolean(args[++i]); + } + if ("-numOfThreadsForCreate".equals(args[i]) && i < args.length - 1) { + numOfThreadsForCreate = Integer.parseInt(args[++i]); + } + // insert task + if ("-numOfRowsPerTable".equals(args[i]) && i < args.length - 1) { + numOfRowsPerTable = Long.parseLong(args[++i]); + } + if ("-numOfThreadsForInsert".equals(args[i]) && i < args.length - 1) { + numOfThreadsForInsert = Integer.parseInt(args[++i]); + } + if ("-numOfTablesPerSQL".equals(args[i]) && i < args.length - 1) { + numOfTablesPerSQL = Integer.parseInt(args[++i]); + } + if ("-numOfValuesPerSQL".equals(args[i]) && i < args.length - 1) { + numOfValuesPerSQL = Integer.parseInt(args[++i]); + } + if ("-startTime".equals(args[i]) && i < args.length - 1) { + startTime = TimeStampUtil.datetimeToLong(args[++i]); + } + if ("-timeGap".equals(args[i]) && i < args.length - 1) { + timeGap = Long.parseLong(args[++i]); + } + if ("-frequency".equals(args[i]) && i < args.length - 1) { + frequency = Integer.parseInt(args[++i]); + } + if ("-order".equals(args[i]) && i < args.length - 1) { + order = Integer.parseInt(args[++i]); + } + if ("-rate".equals(args[i]) && i < args.length - 1) { + rate = Integer.parseInt(args[++i]); + if (rate < 0 || rate > 100) + throw new IllegalArgumentException("rate must between 0 and 100"); + } + if ("-range".equals(args[i]) && i < args.length - 1) { + range = Integer.parseInt(args[++i]); + } + // select task + if ("-executeSql".equals(args[i]) && i < args.length - 1) { + executeSql = args[++i]; + } + + // drop task + if ("-dropTable".equals(args[i]) && i < args.length - 1) { + dropTable = Boolean.parseBoolean(args[++i]); + } + } + } + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JsonConfig.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JsonConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..1c44610095f4b383f82a74dfdc11030a28afb246 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JsonConfig.java @@ -0,0 +1,39 @@ +package com.taosdata.taosdemo.components; + +import com.alibaba.fastjson.JSONObject; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +public class JsonConfig { + + public static void main(String[] args) { + + JsonConfig config = new JsonConfig(); + String str = config.read("insert.json"); + JSONObject jsonObject = JSONObject.parseObject(str); + System.out.println(jsonObject); + + } + + private String read(String fileName) { + try { + BufferedReader reader = new BufferedReader( + new InputStreamReader(JsonConfig.class.getClassLoader().getResourceAsStream(fileName)) + ); + StringBuilder sb = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return sb.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + + return fileName; + } + + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..56e38d2bfce8ba5801d0f8dc48093d198852fbca --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapper.java @@ -0,0 +1,23 @@ +package com.taosdata.taosdemo.dao; + +import java.util.Map; + +public interface DatabaseMapper { + + // create database if not exists XXX + void createDatabase(String dbname); + + // drop database if exists XXX + void dropDatabase(String dbname); + + // create database if not exists XXX keep XX days XX replica XX + void createDatabaseWithParameters(Map map); + + // use XXX + void useDatabase(String dbname); + + //TODO: alter database + + //TODO: show database + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..421a2dea1f4b49786d57b5579ca849976708791e --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java @@ -0,0 +1,47 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; +import java.util.Map; + +public class DatabaseMapperImpl implements DatabaseMapper { + private static final Logger logger = Logger.getLogger(DatabaseMapperImpl.class); + + private final JdbcTemplate jdbcTemplate; + + public DatabaseMapperImpl(DataSource dataSource) { + this.jdbcTemplate = new JdbcTemplate(dataSource); + } + + + @Override + public void createDatabase(String dbname) { + String sql = "create database if not exists " + dbname; + jdbcTemplate.execute(sql); + logger.debug("SQL >>> " + sql); + } + + @Override + public void dropDatabase(String dbname) { + String sql = "drop database if exists " + dbname; + jdbcTemplate.update(sql); + logger.debug("SQL >>> " + sql); + } + + @Override + public void createDatabaseWithParameters(Map map) { + String sql = SqlSpeller.createDatabase(map); + jdbcTemplate.execute(sql); + logger.debug("SQL >>> " + sql); + } + + @Override + public void useDatabase(String dbname) { + String sql = "use " + dbname; + jdbcTemplate.execute(sql); + logger.debug("SQL >>> " + sql); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..e0ddd220c19066afd1cc332f007a82e2fdab2b07 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java @@ -0,0 +1,29 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.SubTableMeta; +import com.taosdata.taosdemo.domain.SubTableValue; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface SubTableMapper { + + // 创建：子表 + void createUsingSuperTable(SubTableMeta subTableMeta); + + // 插入：一张子表多个values + int insertOneTableMultiValues(SubTableValue subTableValue); + + // 插入：一张子表多个values, 自动建表 + int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue); + + // 插入：多张表多个values + int insertMultiTableMultiValues(List tables); + + // 插入：多张表多个values，自动建表 + int insertMultiTableMultiValuesUsingSuperTable(List tables); + + // + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..90b0990a2bf2e9a9bd2738deec17a284c0868280 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java @@ -0,0 +1,81 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.SubTableMeta; +import com.taosdata.taosdemo.domain.SubTableValue; +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; +import java.util.List; + +public class SubTableMapperImpl implements SubTableMapper { + + private static final Logger logger = Logger.getLogger(SubTableMapperImpl.class); + private final JdbcTemplate jdbcTemplate; + + public SubTableMapperImpl(DataSource dataSource) { + this.jdbcTemplate = new JdbcTemplate(dataSource); + } + + @Override + public void createUsingSuperTable(SubTableMeta subTableMeta) { + String sql = SqlSpeller.createTableUsingSuperTable(subTableMeta); + logger.debug("SQL >>> " + sql); + jdbcTemplate.execute(sql); + } + + @Override + public int insertOneTableMultiValues(SubTableValue subTableValue) { + String sql = SqlSpeller.insertOneTableMultiValues(subTableValue); + logger.debug("SQL >>> " + sql); + + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } + + @Override + public int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue) { + String sql = SqlSpeller.insertOneTableMultiValuesUsingSuperTable(subTableValue); + logger.debug("SQL >>> " + sql); + + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } + + @Override + public int insertMultiTableMultiValues(List tables) { + String sql = SqlSpeller.insertMultiSubTableMultiValues(tables); + logger.debug("SQL >>> " + sql); + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } + + @Override + public int insertMultiTableMultiValuesUsingSuperTable(List tables) { + String sql = SqlSpeller.insertMultiTableMultiValuesUsingSuperTable(tables); + logger.debug("SQL >>> " + sql); + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..9f8cec9e8fa5af8741d9efbdce72f240aa7a09aa --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java @@ -0,0 +1,29 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.SuperTableMeta; +import org.springframework.stereotype.Repository; + +@Repository +public interface SuperTableMapper { + + // 创建超级表 create table if not exists xxx.xxx (f1 type1, f2 type2, ... ) tags( t1 type1, t2 type2 ...) + void createSuperTable(SuperTableMeta tableMetadata); + + // 删除超级表 drop table if exists xxx; + void dropSuperTable(String database, String name); + + // + + // + + // + + // + + // + + // + + // + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..efa9a1f39ea41dd1aba65ab610eae095a3164533 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java @@ -0,0 +1,31 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; + +public class SuperTableMapperImpl implements SuperTableMapper { + private static final Logger logger = Logger.getLogger(SuperTableMapperImpl.class); + private JdbcTemplate jdbcTemplate; + + public SuperTableMapperImpl(DataSource dataSource) { + this.jdbcTemplate = new JdbcTemplate(dataSource); + } + + @Override + public void createSuperTable(SuperTableMeta tableMetadata) { + String sql = SqlSpeller.createSuperTable(tableMetadata); + logger.debug("SQL >>> " + sql); + jdbcTemplate.execute(sql); + } + + @Override + public void dropSuperTable(String database, String name) { + String sql = "drop table if exists " + database + "." + name; + logger.debug("SQL >>> " + sql); + jdbcTemplate.execute(sql); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..32d1875e4d1a82f7dfb658d68ed0e63a5cbfa040 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java @@ -0,0 +1,27 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.TableMeta; +import com.taosdata.taosdemo.domain.TableValue; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface TableMapper { + + // 创建：普通表 + void create(TableMeta tableMeta); + + // 插入：一张表多个value + int insertOneTableMultiValues(TableValue values); + + // 插入: 一张表多个value，指定的列 + int insertOneTableMultiValuesWithColumns(TableValue values); + + // 插入：多个表多个value + int insertMultiTableMultiValues(List tables); + + // 插入：多个表多个value, 指定的列 + int insertMultiTableMultiValuesWithColumns(List tables); + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..b049fbe197034ffcd8801b9c4f5e5ff8dbbcc0e0 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java @@ -0,0 +1,49 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.TableMeta; +import com.taosdata.taosdemo.domain.TableValue; +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import java.util.List; + +public class TableMapperImpl implements TableMapper { + private static final Logger logger = Logger.getLogger(TableMapperImpl.class); + private JdbcTemplate template; + + @Override + public void create(TableMeta tableMeta) { + String sql = SqlSpeller.createTable(tableMeta); + logger.debug("SQL >>> " + sql); + template.execute(sql); + } + + @Override + public int insertOneTableMultiValues(TableValue values) { + String sql = SqlSpeller.insertOneTableMultiValues(values); + logger.debug("SQL >>> " + sql); + return template.update(sql); + } + + @Override + public int insertOneTableMultiValuesWithColumns(TableValue values) { + String sql = SqlSpeller.insertOneTableMultiValuesWithColumns(values); + logger.debug("SQL >>> " + sql); + return template.update(sql); + } + + @Override + public int insertMultiTableMultiValues(List tables) { + String sql = SqlSpeller.insertMultiTableMultiValues(tables); + logger.debug("SQL >>> " + sql); + return template.update(sql); + } + + @Override + public int insertMultiTableMultiValuesWithColumns(List tables) { + String sql = SqlSpeller.insertMultiTableMultiValuesWithColumns(tables); + logger.debug("SQL >>> " + sql); + return template.update(sql); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldMeta.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldMeta.java new file mode 100644 index 0000000000000000000000000000000000000000..8a45e99989821562fdf9333b90a2972388cf6003 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldMeta.java @@ -0,0 +1,17 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +@Data +public class FieldMeta { + private String name; + private String type; + + public FieldMeta() { + } + + public FieldMeta(String name, String type) { + this.name = name; + this.type = type; + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldValue.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldValue.java new file mode 100644 index 0000000000000000000000000000000000000000..44805c0d7c2f4e0df60d723afd598f2aa788ee65 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/FieldValue.java @@ -0,0 +1,17 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +@Data +public class FieldValue { + private String name; + private T value; + + public FieldValue() { + } + + public FieldValue(String name, T value) { + this.name = name; + this.value = value; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java new file mode 100644 index 0000000000000000000000000000000000000000..a444fa78dcdeb8f1bb76974a29051c98348a055b --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java @@ -0,0 +1,14 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +import java.util.List; + +@Data +public class RowValue { + private List fields; + + public RowValue(List fields) { + this.fields = fields; + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableMeta.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableMeta.java new file mode 100644 index 0000000000000000000000000000000000000000..81de882448121532a3c840296702d1fb24c37326 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableMeta.java @@ -0,0 +1,15 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +import java.util.List; + +@Data +public class SubTableMeta { + + private String database; + private String supertable; + private String name; + private List tags; + private List fields; +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableValue.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableValue.java new file mode 100644 index 0000000000000000000000000000000000000000..74fb9598bc8e920123180f948b88fb98baf9218e --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SubTableValue.java @@ -0,0 +1,15 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +import java.util.List; + +@Data +public class SubTableValue { + + private String database; + private String supertable; + private String name; + private List tags; + private List values; +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SuperTableMeta.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SuperTableMeta.java new file mode 100644 index 0000000000000000000000000000000000000000..c5c65a4599b776faeead19ffbf065361b293b44b --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/SuperTableMeta.java @@ -0,0 +1,14 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +import java.util.List; + +@Data +public class SuperTableMeta { + + private String database; + private String name; + private List fields; + private List tags; +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableMeta.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableMeta.java new file mode 100644 index 0000000000000000000000000000000000000000..3ab0a75c0b8ddda46f016c93e0a4893ff1ef21fa --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableMeta.java @@ -0,0 +1,13 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +import java.util.List; + +@Data +public class TableMeta { + + private String database; + private String name; + private List fields; +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableValue.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableValue.java new file mode 100644 index 0000000000000000000000000000000000000000..d5502aa46f3508d2f6dcc3668e741adde5b0a4f0 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TableValue.java @@ -0,0 +1,15 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +import java.util.List; + +@Data +public class TableValue { + + private String database; + private String name; + private List columns; + private List values; + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagMeta.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagMeta.java new file mode 100644 index 0000000000000000000000000000000000000000..a385bb4e125e46fe37c1f9542817a4d81301d72f --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagMeta.java @@ -0,0 +1,18 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +@Data +public class TagMeta { + private String name; + private String type; + + public TagMeta() { + + } + + public TagMeta(String name, String type) { + this.name = name; + this.type = type; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagValue.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagValue.java new file mode 100644 index 0000000000000000000000000000000000000000..98ea8c0dc97ebfd330c57f8852eaf2fa379aef50 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/TagValue.java @@ -0,0 +1,17 @@ +package com.taosdata.taosdemo.domain; + +import lombok.Data; + +@Data +public class TagValue { + private String name; + private T value; + + public TagValue() { + } + + public TagValue(String name, T value) { + this.name = name; + this.value = value; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/AbstractService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/AbstractService.java new file mode 100644 index 0000000000000000000000000000000000000000..4afbe9dae82516e050eb7d729d6ecb8eb1e59d6b --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/AbstractService.java @@ -0,0 +1,35 @@ +package com.taosdata.taosdemo.service; + +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class AbstractService { + + protected int getAffectRows(List> futureList) { + int count = 0; + for (Future future : futureList) { + try { + count += future.get(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + return count; + } + + protected int getAffectRows(Future future) { + int count = 0; + try { + count += future.get(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + return count; + } + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java new file mode 100644 index 0000000000000000000000000000000000000000..3c8e9624066bf629a74ebcaa3959a2e15338c363 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java @@ -0,0 +1,42 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.dao.DatabaseMapper; +import com.taosdata.taosdemo.dao.DatabaseMapperImpl; + +import javax.sql.DataSource; +import java.util.Map; + +public class DatabaseService { + + private final DatabaseMapper databaseMapper; + + public DatabaseService(DataSource dataSource) { + this.databaseMapper = new DatabaseMapperImpl(dataSource); + } + + // 建库，指定 name + public void createDatabase(String database) { + databaseMapper.createDatabase(database); + } + + // 建库，指定参数 keep,days,replica等 + public void createDatabase(Map map) { + if (map.isEmpty()) + return; + if (map.containsKey("database") && map.size() == 1) { + createDatabase(map.get("database")); + return; + } + databaseMapper.createDatabaseWithParameters(map); + } + + // drop database + public void dropDatabase(String dbname) { + databaseMapper.dropDatabase(dbname); + } + + // use database + public void useDatabase(String dbname) { + databaseMapper.useDatabase(dbname); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java new file mode 100644 index 0000000000000000000000000000000000000000..efabff6afe904516ad9682cd7197412dc02765ef --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java @@ -0,0 +1,104 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.jdbc.utils.SqlSyntaxValidator; + +import javax.sql.DataSource; +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class QueryService { + + private final DataSource dataSource; + + public QueryService(DataSource dataSource) { + this.dataSource = dataSource; + } + + /* only select or show SQL Statement is valid for executeQuery */ + public Boolean[] areValidQueries(String[] sqls) { + Boolean[] ret = new Boolean[sqls.length]; + for (int i = 0; i < sqls.length; i++) { + ret[i] = true; + if (!SqlSyntaxValidator.isValidForExecuteQuery(sqls[i])) { + ret[i] = false; + continue; + } + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + stmt.executeQuery(sqls[i]); + } catch (SQLException e) { + ret[i] = false; + continue; + } + } + return ret; + } + + public String[] generateSuperTableQueries(String dbName) { + List sqls = new ArrayList<>(); + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + stmt.execute("use " + dbName); + ResultSet rs = stmt.executeQuery("show stables"); + while (rs.next()) { + String name = rs.getString("name"); + sqls.add("select count(*) from " + dbName + "." + name); + sqls.add("select first(*) from " + dbName + "." + name); + sqls.add("select last(*) from " + dbName + "." + name); + sqls.add("select last_row(*) from " + dbName + "." + name); + } + } catch (SQLException e) { + e.printStackTrace(); + } + String[] sqlArr = new String[sqls.size()]; + return sqls.toArray(sqlArr); + } + + public void querySuperTable(String[] sqls, int interval, int threadCount, long queryTimes) { + List threads = IntStream.range(0, threadCount).mapToObj(i -> new Thread(() -> { + // do query + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + long count = queryTimes; + if (count == 0) + count = Long.MAX_VALUE; + while (count > 0) { + for (String sql : sqls) { + long start = System.currentTimeMillis(); + ResultSet rs = stmt.executeQuery(sql); + printResultSet(rs); + long end = System.currentTimeMillis(); + long timecost = end - start; + if (interval - timecost > 0) { + TimeUnit.MILLISECONDS.sleep(interval - timecost); + } + } + count--; + } + + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + + })).collect(Collectors.toList()); + threads.stream().forEach(Thread::start); + for (Thread thread : threads) { + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + private void printResultSet(ResultSet rs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java new file mode 100644 index 0000000000000000000000000000000000000000..ff2e4d0af068a10e62933837817d2d2df0712a4c --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java @@ -0,0 +1,36 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.utils.Printer; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +public class SqlExecuteTask implements Runnable { + private final DataSource dataSource; + private final String sql; + + public SqlExecuteTask(DataSource dataSource, String sql) { + this.dataSource = dataSource; + this.sql = sql; + } + + @Override + public void run() { + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + long start = System.currentTimeMillis(); + boolean execute = stmt.execute(sql); + long end = System.currentTimeMillis(); + if (execute) { + ResultSet rs = stmt.getResultSet(); + Printer.printResult(rs); + } else { + Printer.printSql(sql, true, (end - start)); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java new file mode 100644 index 0000000000000000000000000000000000000000..cea98a1c5d350ed22ed5d26c72fedb212dcb7f26 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java @@ -0,0 +1,208 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.components.JdbcTaosdemoConfig; +import com.taosdata.taosdemo.dao.SubTableMapper; +import com.taosdata.taosdemo.dao.SubTableMapperImpl; +import com.taosdata.taosdemo.domain.SubTableMeta; +import com.taosdata.taosdemo.domain.SubTableValue; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.service.data.SubTableMetaGenerator; +import com.taosdata.taosdemo.service.data.SubTableValueGenerator; +import org.apache.log4j.Logger; + +import javax.sql.DataSource; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class SubTableService extends AbstractService { + + private SubTableMapper mapper; + private static final Logger logger = Logger.getLogger(SubTableService.class); + + public SubTableService(DataSource datasource) { + this.mapper = new SubTableMapperImpl(datasource); + } + + public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable, int numOfThreadsForCreate) { + ExecutorService executor = Executors.newFixedThreadPool(numOfThreadsForCreate); + for (long i = 0; i < numOfTables; i++) { + long tableIndex = i; + executor.execute(() -> createSubTable(superTableMeta, prefixOfTable + (tableIndex + 1))); + } + executor.shutdown(); + try { + executor.awaitTermination(Long.MAX_VALUE,TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public void createSubTable(SuperTableMeta superTableMeta, String tableName) { + // 构造数据 + SubTableMeta meta = SubTableMetaGenerator.generate(superTableMeta, tableName); + createSubTable(meta); + } + + // 创建一张子表，可以指定database，supertable，tablename，tag值 + public void createSubTable(SubTableMeta subTableMeta) { + mapper.createUsingSuperTable(subTableMeta); + } + + /*************************************************************************************************************************/ + // 插入：多线程，多表 + public int insert(List subTableValues, int threadSize, int frequency) { + ExecutorService executor = Executors.newFixedThreadPool(threadSize); + Future future = executor.submit(() -> insert(subTableValues)); + executor.shutdown(); + //TODO：frequency + return getAffectRows(future); + } + + // 插入：单表，insert into xxx values(),()... + public int insert(SubTableValue subTableValue) { + return mapper.insertOneTableMultiValues(subTableValue); + } + + // 插入: 多表，insert into xxx values(),()... xxx values(),()... + public int insert(List subTableValues) { + return mapper.insertMultiTableMultiValues(subTableValues); + } + + // 插入：单表，自动建表, insert into xxx using xxx tags(...) values(),()... + public int insertAutoCreateTable(SubTableValue subTableValue) { + return mapper.insertOneTableMultiValuesUsingSuperTable(subTableValue); + } + + // 插入：多表，自动建表, insert into xxx using XXX tags(...) values(),()... xxx using XXX tags(...) values(),()... + public int insertAutoCreateTable(List subTableValues) { + return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues); + } + + public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime, long gap, JdbcTaosdemoConfig config) { + List taskList = new ArrayList<>(); + List threads = IntStream.range(0, threadSize) + .mapToObj(i -> { + long startInd = i * gap; + long endInd = (i + 1) * gap < tableSize ? (i + 1) * gap : tableSize; + FutureTask task = new FutureTask<>( + new InsertTask(superTableMeta, + startInd, endInd, + startTime, config.timeGap, + config.numOfRowsPerTable, config.numOfTablesPerSQL, config.numOfValuesPerSQL, + config.order, config.rate, config.range, + config.prefixOfTable, config.autoCreateTable) + ); + taskList.add(task); + return new Thread(task, "InsertThread-" + i); + }).collect(Collectors.toList()); + + threads.stream().forEach(Thread::start); + for (Thread thread : threads) { + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + int affectedRows = 0; + for (FutureTask task : taskList) { + try { + affectedRows += task.get(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + + return affectedRows; + } + + private class InsertTask implements Callable { + + private final long startTableInd; // included + private final long endTableInd; // excluded + private final long startTime; + private final long timeGap; + private final long numOfRowsPerTable; + private long numOfTablesPerSQL; + private long numOfValuesPerSQL; + private final SuperTableMeta superTableMeta; + private final int order; + private final int rate; + private final long range; + private final String prefixOfTable; + private final boolean autoCreateTable; + + public InsertTask(SuperTableMeta superTableMeta, long startTableInd, long endTableInd, + long startTime, long timeGap, + long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL, + int order, int rate, long range, + String prefixOfTable, boolean autoCreateTable) { + this.superTableMeta = superTableMeta; + this.startTableInd = startTableInd; + this.endTableInd = endTableInd; + this.startTime = startTime; + this.timeGap = timeGap; + this.numOfRowsPerTable = numOfRowsPerTable; + this.numOfTablesPerSQL = numOfTablesPerSQL; + this.numOfValuesPerSQL = numOfValuesPerSQL; + this.order = order; + this.rate = rate; + this.range = range; + this.prefixOfTable = prefixOfTable; + this.autoCreateTable = autoCreateTable; + } + + + @Override + public Integer call() { + + long numOfTables = endTableInd - startTableInd; + if (numOfRowsPerTable < numOfValuesPerSQL) + numOfValuesPerSQL = (int) numOfRowsPerTable; + if (numOfTables < numOfTablesPerSQL) + numOfTablesPerSQL = (int) numOfTables; + + int affectRows = 0; + // row + for (long rowCnt = 0; rowCnt < numOfRowsPerTable; ) { + long rowSize = numOfValuesPerSQL; + if (rowCnt + rowSize > numOfRowsPerTable) { + rowSize = numOfRowsPerTable - rowCnt; + } + //table + for (long tableCnt = startTableInd; tableCnt < endTableInd; ) { + long tableSize = numOfTablesPerSQL; + if (tableCnt + tableSize > endTableInd) { + tableSize = endTableInd - tableCnt; + } + long startTime = this.startTime + rowCnt * timeGap; +// System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " + rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt + ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: " + timeGap + ""); + /***********************************************/ + // 生成数据 + List data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt, tableSize, rowSize, startTime, timeGap); + // 乱序 + if (order != 0) + SubTableValueGenerator.disrupt(data, rate, range); + // insert + if (autoCreateTable) + affectRows += insertAutoCreateTable(data); + else + affectRows += insert(data); + /***********************************************/ + tableCnt += tableSize; + } + rowCnt += rowSize; + } + + return affectRows; + } + } + + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java new file mode 100644 index 0000000000000000000000000000000000000000..b91348e2d008bc1ac32faffc5912a8509adf42bd --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java @@ -0,0 +1,25 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.dao.SuperTableMapper; +import com.taosdata.taosdemo.dao.SuperTableMapperImpl; +import com.taosdata.taosdemo.domain.SuperTableMeta; + +import javax.sql.DataSource; + +public class SuperTableService { + + private SuperTableMapper superTableMapper; + + public SuperTableService(DataSource dataSource) { + this.superTableMapper = new SuperTableMapperImpl(dataSource); + } + + // 创建超级表，指定每个field的名称和类型，每个tag的名称和类型 + public void create(SuperTableMeta superTableMeta) { + superTableMapper.createSuperTable(superTableMeta); + } + + public void drop(String database, String name) { + superTableMapper.dropSuperTable(database, name); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java new file mode 100644 index 0000000000000000000000000000000000000000..2504fdb0b4cd48ec263d94ec377e1bb8902ea9a7 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java @@ -0,0 +1,25 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.dao.TableMapper; +import com.taosdata.taosdemo.domain.TableMeta; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class TableService extends AbstractService { + + private TableMapper tableMapper; + + //创建一张表 + public void create(TableMeta tableMeta) { + tableMapper.create(tableMeta); + } + + //创建多张表 + public void create(List tables) { + tables.stream().forEach(this::create); + } + + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..73cd981a4651bf589d636b8f57c0af44979f09f1 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java @@ -0,0 +1,48 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.FieldMeta; +import com.taosdata.taosdemo.domain.FieldValue; +import com.taosdata.taosdemo.domain.RowValue; +import com.taosdata.taosdemo.utils.DataGenerator; + +import java.util.*; + +public class FieldValueGenerator { + + public static Random random = new Random(System.currentTimeMillis()); + + // 生成start到end的时间序列，时间戳为顺序，不含有乱序，field的value为随机生成 + public static List generate(long start, long end, long timeGap, List fieldMetaList) { + List values = new ArrayList<>(); + + for (long ts = start; ts < end; ts += timeGap) { + List fieldValues = new ArrayList<>(); + // timestamp + fieldValues.add(new FieldValue(fieldMetaList.get(0).getName(), ts)); + // other values + for (int fieldInd = 1; fieldInd < fieldMetaList.size(); fieldInd++) { + FieldMeta fieldMeta = fieldMetaList.get(fieldInd); + fieldValues.add(new FieldValue(fieldMeta.getName(), DataGenerator.randomValue(fieldMeta.getType()))); + } + values.add(new RowValue(fieldValues)); + } + return values; + } + + // 生成start到end的时间序列，时间戳为顺序，含有乱序，rate为乱序的比例，range为乱序前跳范围，field的value为随机生成 + public static List disrupt(List values, int rate, long range) { + long timeGap = (long) (values.get(1).getFields().get(0).getValue()) - (long) (values.get(0).getFields().get(0).getValue()); + int bugSize = values.size() * rate / 100; + Set bugIndSet = new HashSet<>(); + while (bugIndSet.size() < bugSize) { + bugIndSet.add(random.nextInt(values.size())); + } + for (Integer bugInd : bugIndSet) { + Long timestamp = (Long) values.get(bugInd).getFields().get(0).getValue(); + Long newTimestamp = timestamp - timeGap - random.nextInt((int) range); + values.get(bugInd).getFields().get(0).setValue(newTimestamp); + } + + return values; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..88e3c0d26a5a7558c1c07f0fc38ae21710438dbe --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java @@ -0,0 +1,42 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.SubTableMeta; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.domain.TagValue; + +import java.util.ArrayList; +import java.util.List; + +public class SubTableMetaGenerator { + + // 创建tableSize张子表，使用tablePrefix作为子表名的前缀，使用superTableMeta的元数据 + // create table xxx using XXX tags(XXX) + public static List generate(SuperTableMeta superTableMeta, int tableSize, String tablePrefix) { + List subTableMetaList = new ArrayList<>(); + for (int i = 1; i <= tableSize; i++) { + SubTableMeta subTableMeta = new SubTableMeta(); + // create table xxx.xxx using xxx tags(...) + subTableMeta.setDatabase(superTableMeta.getDatabase()); + subTableMeta.setName(tablePrefix + i); + subTableMeta.setSupertable(superTableMeta.getName()); + subTableMeta.setFields(superTableMeta.getFields()); + List tagValues = TagValueGenerator.generate(superTableMeta.getTags()); + subTableMeta.setTags(tagValues); + subTableMetaList.add(subTableMeta); + } + return subTableMetaList; + } + + public static SubTableMeta generate(SuperTableMeta superTableMeta, String tableName) { + SubTableMeta subTableMeta = new SubTableMeta(); + // create table xxx.xxx using xxx tags(...) + subTableMeta.setDatabase(superTableMeta.getDatabase()); + subTableMeta.setName(tableName); + subTableMeta.setSupertable(superTableMeta.getName()); + subTableMeta.setFields(superTableMeta.getFields()); + List tagValues = TagValueGenerator.generate(superTableMeta.getTags()); + subTableMeta.setTags(tagValues); + return subTableMeta; + } + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..8c318dbd3abf2ddfec8b51e83f32246421c49d51 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java @@ -0,0 +1,100 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.*; +import com.taosdata.taosdemo.utils.TimeStampUtil; +import org.springframework.beans.BeanUtils; + +import java.util.ArrayList; +import java.util.List; + +public class SubTableValueGenerator { + + public static List generate(SuperTableMeta superTableMeta, String prefixOfTables, long tableIndex, long tableSize, long valueSize, long startTime, long timeGap) { + List subTableValues = new ArrayList<>(); + for (int i = 1; i <= tableSize; i++) { + SubTableValue subTableValue = new SubTableValue(); + subTableValue.setDatabase(superTableMeta.getDatabase()); + subTableValue.setName(prefixOfTables + (tableIndex + i)); + subTableValue.setSupertable(superTableMeta.getName()); + TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(startTime, timeGap, valueSize); + List tags = TagValueGenerator.generate(superTableMeta.getTags()); + subTableValue.setTags(tags); + List values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, superTableMeta.getFields()); + subTableValue.setValues(values); + + subTableValues.add(subTableValue); + } + return subTableValues; + } + + public static List generate(List subTableMetaList, int numOfRowsPerTable, long start, long timeGap) { + return generate(subTableMetaList, 0, subTableMetaList.size(), numOfRowsPerTable, start, timeGap); + } + + public static void disrupt(List subTableValueList, int rate, long range) { + subTableValueList.stream().forEach((tableValue) -> { + List values = tableValue.getValues(); + FieldValueGenerator.disrupt(values, rate, range); + }); + } + + public static List> split(List subTableValueList, int numOfTables, int numOfTablesPerSQL, int numOfRowsPerTable, int numOfValuesPerSQL) { + List> dataList = new ArrayList<>(); + if (numOfRowsPerTable < numOfValuesPerSQL) + numOfValuesPerSQL = numOfRowsPerTable; + if (numOfTables < numOfTablesPerSQL) + numOfTablesPerSQL = numOfTables; + //table + for (int tableCnt = 0; tableCnt < numOfTables; ) { + int tableSize = numOfTablesPerSQL; + if (tableCnt + tableSize > numOfTables) { + tableSize = numOfTables - tableCnt; + } + // row + for (int rowCnt = 0; rowCnt < numOfRowsPerTable; ) { + int rowSize = numOfValuesPerSQL; + if (rowCnt + rowSize > numOfRowsPerTable) { + rowSize = numOfRowsPerTable - rowCnt; + } + // System.out.println("rowCnt: " + rowCnt + ", rowSize: " + rowSize + ", tableCnt: " + tableCnt + ", tableSize: " + tableSize); + // split + List blocks = subTableValueList.subList(tableCnt, tableCnt + tableSize); + List newBlocks = new ArrayList<>(); + for (int i = 0; i < blocks.size(); i++) { + SubTableValue subTableValue = blocks.get(i); + SubTableValue newSubTableValue = new SubTableValue(); + BeanUtils.copyProperties(subTableValue, newSubTableValue); + List values = subTableValue.getValues().subList(rowCnt, rowCnt + rowSize); + newSubTableValue.setValues(values); + newBlocks.add(newSubTableValue); + } + dataList.add(newBlocks); + + rowCnt += rowSize; + } + tableCnt += tableSize; + } + return dataList; + } + + public static void main(String[] args) { + split(null, 99, 10, 99, 10); + } + + public static List generate(List subTableMetaList, int tableCnt, int tableSize, int rowSize, long startTime, long timeGap) { + List subTableValueList = new ArrayList<>(); + for (int i = 0; i < tableSize; i++) { + SubTableMeta subTableMeta = subTableMetaList.get(tableCnt + i); + SubTableValue subTableValue = new SubTableValue(); + subTableValue.setDatabase(subTableMeta.getDatabase()); + subTableValue.setName(subTableMeta.getName()); + subTableValue.setSupertable(subTableMeta.getSupertable()); + subTableValue.setTags(subTableMeta.getTags()); + TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(startTime, timeGap, rowSize); + List values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, subTableMeta.getFields()); + subTableValue.setValues(values); + subTableValueList.add(subTableValue); + } + return subTableValueList; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..05aefd01ac8ef5c0bd2d4952e16f557ee7aef8ce --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java @@ -0,0 +1,80 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.FieldMeta; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.domain.TagMeta; +import com.taosdata.taosdemo.utils.TaosConstants; + +import java.util.ArrayList; +import java.util.List; + +public class SuperTableMetaGenerator { + + // 创建超级表，使用指定SQL语句 + public static SuperTableMeta generate(String superTableSQL) { + SuperTableMeta tableMeta = new SuperTableMeta(); + // for example : create table superTable (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int) + superTableSQL = superTableSQL.trim().toLowerCase(); + if (!superTableSQL.startsWith("create")) + throw new RuntimeException("invalid create super table SQL"); + + if (superTableSQL.contains("tags")) { + String tagSQL = superTableSQL.substring(superTableSQL.indexOf("tags") + 4).trim(); + tagSQL = tagSQL.substring(tagSQL.indexOf("(") + 1, tagSQL.lastIndexOf(")")); + String[] tagPairs = tagSQL.split(","); + List tagMetaList = new ArrayList<>(); + for (String tagPair : tagPairs) { + String name = tagPair.trim().split("\\s+")[0]; + String type = tagPair.trim().split("\\s+")[1]; + tagMetaList.add(new TagMeta(name, type)); + } + tableMeta.setTags(tagMetaList); + superTableSQL = superTableSQL.substring(0, superTableSQL.indexOf("tags")); + } + if (superTableSQL.contains("(")) { + String fieldSQL = superTableSQL.substring(superTableSQL.indexOf("(") + 1, superTableSQL.indexOf(")")); + String[] fieldPairs = fieldSQL.split(","); + List fieldList = new ArrayList<>(); + for (String fieldPair : fieldPairs) { + String name = fieldPair.trim().split("\\s+")[0]; + String type = fieldPair.trim().split("\\s+")[1]; + fieldList.add(new FieldMeta(name, type)); + } + tableMeta.setFields(fieldList); + superTableSQL = superTableSQL.substring(0, superTableSQL.indexOf("(")); + } + superTableSQL = superTableSQL.substring(superTableSQL.indexOf("table") + 5).trim(); + if (superTableSQL.contains(".")) { + String database = superTableSQL.split("\\.")[0]; + tableMeta.setDatabase(database); + superTableSQL = superTableSQL.substring(superTableSQL.indexOf(".") + 1); + } + tableMeta.setName(superTableSQL.trim()); + + return tableMeta; + } + + // 创建超级表,指定field和tag的个数 + public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize, String tagPrefix) { + if (fieldSize < 2 || tagSize < 1) { + throw new RuntimeException("create super table but fieldSize less than 2 or tagSize less than 1"); + } + SuperTableMeta tableMetadata = new SuperTableMeta(); + tableMetadata.setDatabase(database); + tableMetadata.setName(name); + // fields + List fields = new ArrayList<>(); + fields.add(new FieldMeta("ts", "timestamp")); + for (int i = 1; i <= fieldSize; i++) { + fields.add(new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length])); + } + tableMetadata.setFields(fields); + // tags + List tags = new ArrayList<>(); + for (int i = 1; i <= tagSize; i++) { + tags.add(new TagMeta(tagPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length])); + } + tableMetadata.setTags(tags); + return tableMetadata; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..b8024fea45c000d7d0944f0ba2d47449d3907d89 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java @@ -0,0 +1,24 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.TagMeta; +import com.taosdata.taosdemo.domain.TagValue; +import com.taosdata.taosdemo.utils.DataGenerator; + +import java.util.ArrayList; +import java.util.List; + +public class TagValueGenerator { + + // 创建标签值：使用tagMetas + public static List generate(List tagMetas) { + List tagValues = new ArrayList<>(); + for (int i = 0; i < tagMetas.size(); i++) { + TagMeta tagMeta = tagMetas.get(i); + TagValue tagValue = new TagValue(); + tagValue.setName(tagMeta.getName()); + tagValue.setValue(DataGenerator.randomValue(tagMeta.getType())); + tagValues.add(tagValue); + } + return tagValues; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/DataGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/DataGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..a200d17ef6c4c4987160c9966a50737773153bc9 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/DataGenerator.java @@ -0,0 +1,120 @@ +package com.taosdata.taosdemo.utils; + +import java.util.Random; + +public class DataGenerator { + private static Random random = new Random(System.currentTimeMillis()); + private static final String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; + + // "timestamp", "int", "bigint", "float", "double", "binary(64)", "smallint", "tinyint", "bool", "nchar(64)", + + public static Object randomValue(String type) { + int length = 64; + if (type.contains("(")) { + length = Integer.parseInt(type.substring(type.indexOf("(") + 1, type.indexOf(")"))); + type = type.substring(0, type.indexOf("(")); + } + switch (type.trim().toLowerCase()) { + case "timestamp": + return randomTimestamp(); + case "int": + return randomInt(); + case "bigint": + return randomBigint(); + case "float": + return randomFloat(); + case "double": + return randomDouble(); + case "binary": + return randomBinary(length); + case "smallint": + return randomSmallint(); + case "tinyint": + return randomTinyint(); + case "bool": + return randomBoolean(); + case "nchar": + return randomNchar(length); + default: + throw new IllegalArgumentException("Unexpected value: " + type); + } + } + + public static Long randomTimestamp() { + long start = System.currentTimeMillis(); + return randomTimestamp(start, start + 60l * 60l * 1000l); + } + + public static Long randomTimestamp(Long start, Long end) { + return start + (long) random.nextInt((int) (end - start)); + } + + public static String randomNchar(int length) { + return randomChinese(length); + } + + public static Boolean randomBoolean() { + return random.nextBoolean(); + } + + public static Integer randomTinyint() { + return randomInt(-127, 127); + } + + public static Integer randomSmallint() { + return randomInt(-32767, 32767); + } + + public static String randomBinary(int length) { + return randomString(length); + } + + public static String randomString(int length) { + String zh_en = ""; + for (int i = 0; i < length; i++) { + zh_en += alphabet.charAt(random.nextInt(alphabet.length())); + } + return zh_en; + } + + public static String randomChinese(int length) { + String zh_cn = ""; + int bottom = Integer.parseInt("4e00", 16); + int top = Integer.parseInt("9fa5", 16); + + for (int i = 0; i < length; i++) { + char c = (char) (random.nextInt(top - bottom + 1) + bottom); + zh_cn += new String(new char[]{c}); + } + return zh_cn; + } + + public static Double randomDouble() { + return randomDouble(0, 100); + } + + public static Double randomDouble(double bottom, double top) { + return bottom + (top - bottom) * random.nextDouble(); + } + + public static Float randomFloat() { + return randomFloat(0, 100); + } + + public static Float randomFloat(float bottom, float top) { + return bottom + (top - bottom) * random.nextFloat(); + } + + public static Long randomBigint() { + return random.nextLong(); + } + + public static Integer randomInt(int bottom, int top) { + return bottom + random.nextInt((top - bottom)); + } + + public static Integer randomInt() { + return randomInt(0, 100); + } + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java new file mode 100644 index 0000000000000000000000000000000000000000..a4627463ec5cd1bbccdb64b67506ba38f712de8f --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java @@ -0,0 +1,27 @@ +package com.taosdata.taosdemo.utils; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +public class Printer { + + public static void printResult(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + System.out.printf("%s: %s\t", columnLabel, value); + } + System.out.println(); + } + } + + public static void printSql(String sql, boolean succeed, long cost) { + System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private Printer() { + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java new file mode 100644 index 0000000000000000000000000000000000000000..a60f0641d3a4441195c3a60639fbe3a197115dc3 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java @@ -0,0 +1,194 @@ +package com.taosdata.taosdemo.utils; + +import com.taosdata.taosdemo.domain.*; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class SqlSpeller { + + // create database if not exists xxx keep xx days xx replica xx cache xx... + public static String createDatabase(Map map) { + StringBuilder sb = new StringBuilder(); + sb.append("create database if not exists ").append(map.get("database")).append(" "); + if (map.containsKey("keep")) + sb.append("keep ").append(map.get("keep")).append(" "); + if (map.containsKey("days")) + sb.append("days ").append(map.get("days")).append(" "); + if (map.containsKey("replica")) + sb.append("replica ").append(map.get("replica")).append(" "); + if (map.containsKey("cache")) + sb.append("cache ").append(map.get("cache")).append(" "); + if (map.containsKey("blocks")) + sb.append("blocks ").append(map.get("blocks")).append(" "); + if (map.containsKey("minrows")) + sb.append("minrows ").append(map.get("minrows")).append(" "); + if (map.containsKey("maxrows")) + sb.append("maxrows ").append(map.get("maxrows")).append(" "); + if (map.containsKey("precision")) + sb.append("precision ").append(map.get("precision")).append(" "); + if (map.containsKey("comp")) + sb.append("comp ").append(map.get("comp")).append(" "); + if (map.containsKey("walLevel")) + sb.append("walLevel ").append(map.get("walLevel")).append(" "); + if (map.containsKey("quorum")) + sb.append("quorum ").append(map.get("quorum")).append(" "); + if (map.containsKey("fsync")) + sb.append("fsync ").append(map.get("fsync")).append(" "); + if (map.containsKey("update")) + sb.append("update ").append(map.get("update")).append(" "); + return sb.toString(); + } + + // create table if not exists xx.xx using xx.xx tags(x,x,x) + public static String createTableUsingSuperTable(SubTableMeta subTableMeta) { + StringBuilder sb = new StringBuilder(); + sb.append("create table if not exists ").append(subTableMeta.getDatabase()).append(".").append(subTableMeta.getName()).append(" "); + sb.append("using ").append(subTableMeta.getDatabase()).append(".").append(subTableMeta.getSupertable()).append(" "); +// String tagStr = subTableMeta.getTags().stream().filter(Objects::nonNull) +// .map(tagValue -> tagValue.getName() + " '" + tagValue.getValue() + "' ") +// .collect(Collectors.joining(",", "(", ")")); + sb.append("tags ").append(tagValues(subTableMeta.getTags())); + return sb.toString(); + } + + // insert into xx.xxx values(x,x,x),(x,x,x)... + public static String insertOneTableMultiValues(SubTableValue subTableValue) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getName() + " "); + sb.append("values").append(rowValues(subTableValue.getValues())); + return sb.toString(); + } + + //f1, f2, f3 + private static String fieldValues(List fields) { + return IntStream.range(0, fields.size()).mapToObj(i -> { + if (i == 0) { + return "" + fields.get(i).getValue() + ""; + } else { + return "'" + fields.get(i).getValue() + "'"; + } + }).collect(Collectors.joining(",", "(", ")")); + +// return fields.stream() +// .filter(Objects::nonNull) +// .map(fieldValue -> "'" + fieldValue.getValue() + "'") +// .collect(Collectors.joining(",", "(", ")")); + } + + //(f1, f2, f3),(f1, f2, f3) + private static String rowValues(List rowValues) { + return rowValues.stream().filter(Objects::nonNull) + .map(rowValue -> fieldValues(rowValue.getFields())) + .collect(Collectors.joining(",", "", "")); + } + + // insert into xx.xxx using xx.xx tags(x,x,x) values(x,x,x),(x,x,x)... + public static String insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getName()).append(" "); + sb.append("using ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getSupertable()).append(" "); + sb.append("tags ").append(tagValues(subTableValue.getTags()) + " "); + sb.append("values ").append(rowValues(subTableValue.getValues())); + return sb.toString(); + } + + // (t1,t2,t3...) + private static String tagValues(List tags) { + return tags.stream().filter(Objects::nonNull) + .map(tagValue -> "'" + tagValue.getValue() + "'") + .collect(Collectors.joining(",", "(", ")")); + } + + // insert into xx.xx values(),(),()... xx.xx values(),()... + public static String insertMultiSubTableMultiValues(List tables) { + return "insert into " + tables.stream().filter(Objects::nonNull) + .map(table -> table.getDatabase() + "." + table.getName() + " values " + rowValues(table.getValues())) + .collect(Collectors.joining(" ", "", "")); + } + + // insert into xx.xx using xx.xx tags(xx,xx) values(),()... + public static String insertMultiTableMultiValuesUsingSuperTable(List tables) { + return "insert into " + tables.stream().filter(Objects::nonNull) + .map(table -> { + StringBuilder sb = new StringBuilder(); + sb.append(table.getDatabase()).append(".").append(table.getName()); + sb.append(" using ").append(table.getDatabase()).append(".").append(table.getSupertable()); + sb.append(" tags ").append(tagValues(table.getTags())); + sb.append(" values ").append(rowValues(table.getValues())); + return sb.toString(); + }).collect(Collectors.joining(" ")); + } + + // create table if not exists xx.xx (f1 xx,f2 xx...) tags(t1 xx, t2 xx...) + public static String createSuperTable(SuperTableMeta tableMetadata) { + StringBuilder sb = new StringBuilder(); + sb.append("create table if not exists ").append(tableMetadata.getDatabase()).append(".").append(tableMetadata.getName()); + String fields = tableMetadata.getFields().stream() + .filter(Objects::nonNull).map(field -> field.getName() + " " + field.getType() + " ") + .collect(Collectors.joining(",", "(", ")")); + sb.append(fields); + sb.append(" tags "); + String tags = tableMetadata.getTags().stream().filter(Objects::nonNull) + .map(tag -> tag.getName() + " " + tag.getType() + " ") + .collect(Collectors.joining(",", "(", ")")); + sb.append(tags); + return sb.toString(); + } + + + public static String createTable(TableMeta tableMeta) { + StringBuilder sb = new StringBuilder(); + sb.append("create table if not exists ").append(tableMeta.getDatabase()).append(".").append(tableMeta.getName()).append(" "); + String fields = tableMeta.getFields().stream() + .filter(Objects::nonNull).map(field -> field.getName() + " " + field.getType() + " ") + .collect(Collectors.joining(",", "(", ")")); + sb.append(fields); + return sb.toString(); + } + + // insert into xx.xx values() + public static String insertOneTableMultiValues(TableValue table) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table.getDatabase()).append(".").append(table.getName() + " "); + sb.append("values").append(rowValues(table.getValues())); + return sb.toString(); + + } + + // insert into xx.xx (f1, f2, f3...) values(xx,xx,xx),(xx,xx,xx)... + public static String insertOneTableMultiValuesWithColumns(TableValue table) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table.getDatabase()).append(".").append(table.getName()).append(" "); + sb.append(columnNames(table.getColumns())); + sb.append(" values ").append(rowValues(table.getValues())); + return sb.toString(); + } + + // (f1, f2, f3...) + private static String columnNames(List fields) { + return fields.stream() + .filter(Objects::nonNull) + .map(column -> column.getName() + " ") + .collect(Collectors.joining(",", "(", ")")); + } + + public static String insertMultiTableMultiValuesWithColumns(List tables) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(tables.stream().filter(Objects::nonNull) + .map(table -> table.getDatabase() + "." + table.getName() + " " + columnNames(table.getColumns()) + " values " + rowValues(table.getValues())) + .collect(Collectors.joining(" "))); + return sb.toString(); + } + + public static String insertMultiTableMultiValues(List tables) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(tables.stream().filter(Objects::nonNull).map(table -> + table.getDatabase() + "." + table.getName() + " values " + rowValues(table.getValues()) + ).collect(Collectors.joining(" "))); + return sb.toString(); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TaosConstants.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TaosConstants.java new file mode 100644 index 0000000000000000000000000000000000000000..23c3c5279a15a1dbe529c66398d1378c826ce99a --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TaosConstants.java @@ -0,0 +1,8 @@ +package com.taosdata.taosdemo.utils; + +public class TaosConstants { + public static final String[] DATA_TYPES = { + "timestamp", "int", "bigint", "float", "double", + "binary(64)", "smallint", "tinyint", "bool", "nchar(64)", + }; +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..9cfce16d828510be2c62a9a383e8e3d3badb622e --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java @@ -0,0 +1,67 @@ +package com.taosdata.taosdemo.utils; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class TimeStampUtil { + + private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; + + public static long datetimeToLong(String dateTime) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + try { + return sdf.parse(dateTime).getTime(); + } catch (ParseException e) { + throw new IllegalArgumentException("invalid datetime string >>> " + dateTime); + } + } + + public static String longToDatetime(long time) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + return sdf.format(new Date(time)); + } + + public static class TimeTuple { + public Long start; + public Long end; + public Long timeGap; + + TimeTuple(long start, long end, long timeGap) { + this.start = start; + this.end = end; + this.timeGap = timeGap; + } + } + + public static TimeTuple range(long start, long timeGap, long size) { + long now = System.currentTimeMillis(); + if (timeGap < 1) + timeGap = 1; + if (start == 0) + start = now - size * timeGap; + + // 如果size小于1异常 + if (size < 1) + throw new IllegalArgumentException("size less than 1."); + // 如果timeGap为1，已经超长，需要前移start + if (start + size > now) { + start = now - size; + return new TimeTuple(start, now, 1); + } + long end = start + (long) (timeGap * size); + if (end > now) { + //压缩timeGap + end = now; + double gap = (end - start) / (size * 1.0f); + if (gap < 1.0f) { + timeGap = 1; + start = end - size; + } else { + timeGap = (long) gap; + end = start + (long) (timeGap * size); + } + } + return new TimeTuple(start, end, timeGap); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/application.properties b/tests/examples/JDBC/taosdemo/src/main/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..488185196f1d2325fd9896d30068cbb202180a3f --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/resources/application.properties @@ -0,0 +1,5 @@ +jdbc.driver=com.taosdata.jdbc.rs.RestfulDriver +#jdbc.driver=com.taosdata.jdbc.TSDBDriver +hikari.maximum-pool-size=20 +hikari.minimum-idle=20 +hikari.max-lifetime=0 \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..a7bd87e6d3bbf9f6ec1b0a68d31c4da6c620c994 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json @@ -0,0 +1,119 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 2, + "databases": [ + { + "dbinfo": { + "name": "db04", + "drop": "no", + "replica": 1, + "days": 2, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "quorum": 1, + "fsync": 3000, + "update": 0 + }, + "super_tables": [ + { + "name": "stb04", + "child_table_exists": "no", + "childtable_count": 10, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 100, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 3, + "max_sql_len": 1024, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + { + "type": "TINYINT" + }, + { + "type": "SMALLINT" + }, + { + "type": "INT" + }, + { + "type": "BIGINT" + }, + { + "type": "BOOL" + }, + { + "type": "FLOAT" + }, + { + "type": "DOUBLE" + }, + { + "type": "TIMESTAMP" + }, + { + "type": "BINARY", + "len": 16 + }, + { + "type": "NCHAR", + "len": 4 + } + ], + "tags": [ + { + "type": "TINYINT" + }, + { + "type": "SMALLINT" + }, + { + "type": "INT" + }, + { + "type": "BIGINT" + }, + { + "type": "BOOL" + }, + { + "type": "FLOAT" + }, + { + "type": "DOUBLE" + }, + { + "type": "BINARY", + "len": 16 + }, + { + "type": "NCHAR", + "len": 4 + } + ] + } + ] + } + ] +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties b/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties similarity index 82% rename from tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties rename to tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties index b445e5f52edf1a2b19c35c1be187716d17cdd7b1..352545854d0f68f07e4da6a98504ffa7754b03b1 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties +++ b/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties @@ -1,18 +1,18 @@ ### 设置### -log4j.rootLogger=debug,stdout,DebugLog,ErrorLog +log4j.rootLogger=info,stdout ### 输出信息到控制抬 ### log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n -### 输出DEBUG 级别以上的日志到=logs/error.log ### +### 输出DEBUG 级别以上的日志到=logs/debug.log log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.DebugLog.File=logs/debug.log log4j.appender.DebugLog.Append=true log4j.appender.DebugLog.Threshold=DEBUG log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n -### 输出ERROR 级别以上的日志到=logs/error.log ### +### 输出ERROR 级别以上的日志到=logs/error.log log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.ErrorLog.File=logs/error.log log4j.appender.ErrorLog.Append=true diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/query.json b/tests/examples/JDBC/taosdemo/src/main/resources/query.json new file mode 100644 index 0000000000000000000000000000000000000000..cc6900d77c3941e6af3274efdfe782c42a557990 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/resources/query.json @@ -0,0 +1,17 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db01", + "super_table_query": + {"rate":1, "concurrent":1,"time":10000, + "sqls": [{"sql": "select count(*) from stb01", "result": "./query_res0.txt"}] + }, + "sub_table_query": + {"stblname": "stb01", "rate":1, "threads":1, + "sqls": [{"sql": "select count(*) from xxxx", "result": "./query_res1.txt"}] + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/templates/index.html b/tests/examples/JDBC/taosdemo/src/main/resources/templates/index.html new file mode 100644 index 0000000000000000000000000000000000000000..953a058032612c7eac8eacc68f738d5fef03ec39 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/resources/templates/index.html @@ -0,0 +1,10 @@ + + + + + Index + + +

    Developing~~~

    + + \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..621ba7df5da4bb29747a5a4af1f91d51f6a1d7d4 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java @@ -0,0 +1,23 @@ +package com.taosdata.taosdemo.service; + +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +public class DatabaseServiceTest { + private DatabaseService service; + + @Test + public void testCreateDatabase1() { + service.createDatabase("testXXXX"); + } + + @Test + public void dropDatabase() { + service.dropDatabase("testXXXX"); + } + + @Test + public void useDatabase() { + service.useDatabase("test"); + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f2ad25710c1a82136d6316ed69e379bc3925897d --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java @@ -0,0 +1,41 @@ +package com.taosdata.taosdemo.service; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.BeforeClass; +import org.junit.Test; + +public class QueryServiceTest { + private static QueryService queryService; + + @Test + public void areValidQueries() { + + } + + @Test + public void generateSuperTableQueries() { + String[] sqls = queryService.generateSuperTableQueries("restful_test"); + for (String sql : sqls) { + System.out.println(sql); + } + } + + @Test + public void querySuperTable() { + String[] sqls = queryService.generateSuperTableQueries("restful_test"); + queryService.querySuperTable(sqls, 1000, 10, 10); + } + + @BeforeClass + public static void beforeClass() throws ClassNotFoundException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + config.setUsername("root"); + config.setPassword("taosdata"); + HikariDataSource dataSource = new HikariDataSource(config); + queryService = new QueryService(dataSource); + } + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SubTableServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SubTableServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f7e5cd45057472602ad6e7a43a8d8bdb31a02b40 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SubTableServiceTest.java @@ -0,0 +1,42 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.domain.SubTableMeta; +import com.taosdata.taosdemo.domain.TagValue; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class SubTableServiceTest { + + private SubTableService service; + + private List subTables; + + @Before + public void before() { + subTables = new ArrayList<>(); + for (int i = 1; i <= 1; i++) { + SubTableMeta subTableMeta = new SubTableMeta(); + subTableMeta.setDatabase("test"); + subTableMeta.setSupertable("weather"); + subTableMeta.setName("t" + i); + List tags = new ArrayList<>(); + tags.add(new TagValue("location", "beijing")); + tags.add(new TagValue("groupId", i)); + subTableMeta.setTags(tags); + subTables.add(subTableMeta); + } + } + + @Test + public void testCreateSubTable() { + + } + + @Test + public void testCreateSubTableList() { + + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..33e52af1eaa779e7ed402a63633bf0dbb9fbadd7 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java @@ -0,0 +1,32 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.domain.FieldMeta; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.domain.TagMeta; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class SuperTableServiceTest { + + private SuperTableService service; + + @Test + public void testCreate() { + SuperTableMeta superTableMeta = new SuperTableMeta(); + superTableMeta.setDatabase("test"); + superTableMeta.setName("weather"); + List fields = new ArrayList<>(); + fields.add(new FieldMeta("ts", "timestamp")); + fields.add(new FieldMeta("temperature", "float")); + fields.add(new FieldMeta("humidity", "int")); + superTableMeta.setFields(fields); + List tags = new ArrayList<>(); + tags.add(new TagMeta("location", "nchar(64)")); + tags.add(new TagMeta("groupId", "int")); + superTableMeta.setTags(tags); + service.create(superTableMeta); + } + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..1f52198d68823326dd81d8c419fc02d89e15ef2d --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java @@ -0,0 +1,31 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.domain.TableMeta; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class TableServiceTest { + private TableService tableService; + + private List tables; + + @Before + public void before() { + tables = new ArrayList<>(); + for (int i = 0; i < 1; i++) { + TableMeta tableMeta = new TableMeta(); + tableMeta.setDatabase("test"); + tableMeta.setName("weather" + (i + 1)); + tables.add(tableMeta); + } + } + + @Test + public void testCreate() { + tableService.create(tables); + } + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/FieldValueGeneratorTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/FieldValueGeneratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..aea3cc76ca74951a6bbcc91fe3c6b5c7e078f0df --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/FieldValueGeneratorTest.java @@ -0,0 +1,59 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.FieldMeta; +import com.taosdata.taosdemo.domain.RowValue; +import com.taosdata.taosdemo.utils.TimeStampUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class FieldValueGeneratorTest { + + private List rowValues; + + @Test + public void generate() { + List fieldMetas = new ArrayList<>(); + fieldMetas.add(new FieldMeta("ts", "timestamp")); + fieldMetas.add(new FieldMeta("temperature", "float")); + fieldMetas.add(new FieldMeta("humidity", "int")); + + long start = TimeStampUtil.datetimeToLong("2020-01-01 00:00:00.000"); + long end = TimeStampUtil.datetimeToLong("2020-01-01 10:00:00.000"); + + rowValues = FieldValueGenerator.generate(start, end, 1000l * 3600, fieldMetas); + Assert.assertEquals(10, rowValues.size()); + } + + @Test + public void disrupt() { + List fieldMetas = new ArrayList<>(); + fieldMetas.add(new FieldMeta("ts", "timestamp")); + fieldMetas.add(new FieldMeta("temperature", "float")); + fieldMetas.add(new FieldMeta("humidity", "int")); + + long start = TimeStampUtil.datetimeToLong("2020-01-01 00:00:00.000"); + long end = TimeStampUtil.datetimeToLong("2020-01-01 10:00:00.000"); + + rowValues = FieldValueGenerator.generate(start, end, 1000l * 3600l, fieldMetas); + + FieldValueGenerator.disrupt(rowValues, 20, 1000); + Assert.assertEquals(10, rowValues.size()); + } + + @After + public void after() { + for (RowValue row : rowValues) { + row.getFields().stream().forEach(field -> { + if (field.getName().equals("ts")) { + System.out.print(TimeStampUtil.longToDatetime((Long) field.getValue())); + } else + System.out.print(" ," + field.getValue()); + }); + System.out.println(); + } + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/SubTableMetaGeneratorTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/SubTableMetaGeneratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..78c8e9283f9a045b251419e5a07b1c97bd105254 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/SubTableMetaGeneratorTest.java @@ -0,0 +1,52 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.FieldMeta; +import com.taosdata.taosdemo.domain.SubTableMeta; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.domain.TagMeta; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class SubTableMetaGeneratorTest { + List subTableMetas; + + @Test + public void generate() { + SuperTableMeta superTableMeta = new SuperTableMeta(); + superTableMeta.setDatabase("test"); + superTableMeta.setName("weather"); + List fields = new ArrayList<>(); + fields.add(new FieldMeta("ts", "timestamp")); + fields.add(new FieldMeta("temperature", "float")); + fields.add(new FieldMeta("humidity", "int")); + superTableMeta.setFields(fields); + List tags = new ArrayList<>(); + tags.add(new TagMeta("location", "nchar(64)")); + tags.add(new TagMeta("groupId", "int")); + superTableMeta.setTags(tags); + + subTableMetas = SubTableMetaGenerator.generate(superTableMeta, 10, "t"); + Assert.assertEquals(10, subTableMetas.size()); + Assert.assertEquals("t1", subTableMetas.get(0).getName()); + Assert.assertEquals("t2", subTableMetas.get(1).getName()); + Assert.assertEquals("t3", subTableMetas.get(2).getName()); + Assert.assertEquals("t4", subTableMetas.get(3).getName()); + Assert.assertEquals("t5", subTableMetas.get(4).getName()); + Assert.assertEquals("t6", subTableMetas.get(5).getName()); + Assert.assertEquals("t7", subTableMetas.get(6).getName()); + Assert.assertEquals("t8", subTableMetas.get(7).getName()); + Assert.assertEquals("t9", subTableMetas.get(8).getName()); + Assert.assertEquals("t10", subTableMetas.get(9).getName()); + } + + @After + public void after() { + for (SubTableMeta subTableMeta : subTableMetas) { + System.out.println(subTableMeta); + } + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/SuperTableMetaGeneratorImplTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/SuperTableMetaGeneratorImplTest.java new file mode 100644 index 0000000000000000000000000000000000000000..11c5312cf6c9b4893681728115519509985a42a2 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/SuperTableMetaGeneratorImplTest.java @@ -0,0 +1,60 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.FieldMeta; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.domain.TagMeta; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class SuperTableMetaGeneratorImplTest { + private SuperTableMeta meta; + + @Test + public void generate() { + String sql = "create table test.weather (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; + meta = SuperTableMetaGenerator.generate(sql); + Assert.assertEquals("test", meta.getDatabase()); + Assert.assertEquals("weather", meta.getName()); + Assert.assertEquals(3, meta.getFields().size()); + Assert.assertEquals("ts", meta.getFields().get(0).getName()); + Assert.assertEquals("timestamp", meta.getFields().get(0).getType()); + Assert.assertEquals("temperature", meta.getFields().get(1).getName()); + Assert.assertEquals("float", meta.getFields().get(1).getType()); + Assert.assertEquals("humidity", meta.getFields().get(2).getName()); + Assert.assertEquals("int", meta.getFields().get(2).getType()); + + Assert.assertEquals("location", meta.getTags().get(0).getName()); + Assert.assertEquals("nchar(64)", meta.getTags().get(0).getType()); + Assert.assertEquals("groupid", meta.getTags().get(1).getName()); + Assert.assertEquals("int", meta.getTags().get(1).getType()); + } + + @Test + public void generate2() { + meta = SuperTableMetaGenerator.generate("test", "weather", 10, "col", 10, "tag"); + Assert.assertEquals("test", meta.getDatabase()); + Assert.assertEquals("weather", meta.getName()); + Assert.assertEquals(11, meta.getFields().size()); + for (FieldMeta fieldMeta : meta.getFields()) { + Assert.assertNotNull(fieldMeta.getName()); + Assert.assertNotNull(fieldMeta.getType()); + } + for (TagMeta tagMeta : meta.getTags()) { + Assert.assertNotNull(tagMeta.getName()); + Assert.assertNotNull(tagMeta.getType()); + } + } + + @After + public void after() { + System.out.println(meta.getDatabase()); + System.out.println(meta.getName()); + for (FieldMeta fieldMeta : meta.getFields()) { + System.out.println(fieldMeta); + } + for (TagMeta tagMeta : meta.getTags()) { + System.out.println(tagMeta); + } + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/TagValueGeneratorTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/TagValueGeneratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..37c9051c941b3540fd24371cff20be426d4fecba --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/data/TagValueGeneratorTest.java @@ -0,0 +1,37 @@ +package com.taosdata.taosdemo.service.data; + +import com.taosdata.taosdemo.domain.TagMeta; +import com.taosdata.taosdemo.domain.TagValue; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class TagValueGeneratorTest { + List tagvalues; + + @Test + public void generate() { + List tagMetaList = new ArrayList<>(); + tagMetaList.add(new TagMeta("location", "nchar(10)")); + tagMetaList.add(new TagMeta("groupId", "int")); + tagMetaList.add(new TagMeta("ts", "timestamp")); + tagMetaList.add(new TagMeta("temperature", "float")); + tagMetaList.add(new TagMeta("humidity", "double")); + tagMetaList.add(new TagMeta("text", "binary(10)")); + tagvalues = TagValueGenerator.generate(tagMetaList); + Assert.assertEquals("location", tagvalues.get(0).getName()); + Assert.assertEquals("groupId", tagvalues.get(1).getName()); + Assert.assertEquals("ts", tagvalues.get(2).getName()); + Assert.assertEquals("temperature", tagvalues.get(3).getName()); + Assert.assertEquals("humidity", tagvalues.get(4).getName()); + Assert.assertEquals("text", tagvalues.get(5).getName()); + } + + @After + public void after() { + tagvalues.stream().forEach(System.out::println); + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/DataGeneratorTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/DataGeneratorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..7d12782526e4d64e120ba5e4c72809846356cd00 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/DataGeneratorTest.java @@ -0,0 +1,20 @@ +package com.taosdata.taosdemo.utils; + +import org.junit.Assert; +import org.junit.Test; + +public class DataGeneratorTest { + + @Test + public void randomValue() { + for (int i = 0; i < TaosConstants.DATA_TYPES.length; i++) { + System.out.println(TaosConstants.DATA_TYPES[i] + " >>> " + DataGenerator.randomValue(TaosConstants.DATA_TYPES[i])); + } + } + + @Test + public void randomNchar() { + String s = DataGenerator.randomNchar(10); + Assert.assertEquals(10, s.length()); + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/SqlSpellerTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/SqlSpellerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..daabd51ca75a6c9f4bfeead0b747c4de69f40a7a --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/SqlSpellerTest.java @@ -0,0 +1,254 @@ +package com.taosdata.taosdemo.utils; + +import com.taosdata.taosdemo.domain.*; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class SqlSpellerTest { + + @Test + public void createDatabase() { + HashMap map = new HashMap<>(); + map.put("database", "jdbcdb"); + map.put("keep", "3650"); + map.put("days", "30"); + map.put("replica", "1"); + map.put("minRows", "100"); + map.put("maxRows", "1000"); + map.put("cache", "16"); + map.put("blocks", "8"); + map.put("precision", "ms"); + map.put("comp", "2"); + map.put("walLevel", "1"); + map.put("quorum", "1"); + map.put("fsync", "3000"); + map.put("update", "0"); + String sql = SqlSpeller.createDatabase(map); + System.out.println(sql); + } + + @Test + public void createTableUsingSuperTable() { + SubTableMeta subTableMeta = new SubTableMeta(); + subTableMeta.setDatabase("test"); + subTableMeta.setSupertable("weather"); + subTableMeta.setName("t1"); + List tags = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + tags.add(new TagValue("tag" + (i + 1), "nchar(64)")); + } + subTableMeta.setTags(tags); + String sql = SqlSpeller.createTableUsingSuperTable(subTableMeta); + System.out.println(sql); + } + + @Test + public void insertOneTableMultiValues() { + String sql = SqlSpeller.insertOneTableMultiValues(tables.get(0)); + System.out.println(sql); + } + + @Test + public void insertOneTableMultiValuesUsingSuperTable() { + String sql = SqlSpeller.insertOneTableMultiValuesUsingSuperTable(tables.get(0)); + System.out.println(sql); + } + + @Test + public void insertMultiTableMultiValues() { + String sql = SqlSpeller.insertMultiSubTableMultiValues(tables); + System.out.println(sql); + } + + @Test + public void insertMultiTableMultiValuesUsingSuperTable() { + String sql = SqlSpeller.insertMultiTableMultiValuesUsingSuperTable(tables); + System.out.println(sql); + } + + private List tables; + + @Before + public void before() { + tables = new ArrayList<>(); + for (int ind = 0; ind < 3; ind++) { + SubTableValue table = new SubTableValue(); + table.setDatabase("test"); + // supertable + table.setSupertable("weather"); + table.setName("t" + (ind + 1)); + // tags + List tags = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + tags.add(new TagValue("tag" + (i + 1), "beijing")); + } + table.setTags(tags); + // values + List values = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + List fields = new ArrayList<>(); + for (int j = 0; j < 4; j++) { + fields.add(new FieldValue("f" + (j + 1), (j + 1) * 10)); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + + tables.add(table); + } + } + + @Test + public void createSuperTable() { + SuperTableMeta superTableMeta = new SuperTableMeta(); + superTableMeta.setDatabase("test"); + superTableMeta.setName("weather"); + List fields = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + fields.add(new FieldMeta("f" + (i + 1), "int")); + } + superTableMeta.setFields(fields); + List tags = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + tags.add(new TagMeta("t" + (i + 1), "nchar(64)")); + } + superTableMeta.setTags(tags); + + String sql = SqlSpeller.createSuperTable(superTableMeta); + System.out.println(sql); + } + + @Test + public void createTable() { + TableMeta table = new TableMeta(); + table.setDatabase("test"); + table.setName("t1"); + List fields = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + FieldMeta field = new FieldMeta(); + field.setName("f" + (i + 1)); + field.setType("nchar(64)"); + fields.add(field); + } + table.setFields(fields); + String sql = SqlSpeller.createTable(table); + System.out.println(sql); + } + + + @Test + public void testInsertOneTableMultiValues() { + TableValue table = new TableValue(); + table.setDatabase("test"); + table.setName("t1"); + List values = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + List fields = new ArrayList<>(); + for (int k = 0; k < 2; k++) { + FieldValue field = new FieldValue<>(); + field.setValue((k + 1) * 100); + fields.add(field); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + + String sql = SqlSpeller.insertOneTableMultiValues(table); + System.out.println(sql); + } + + @Test + public void insertOneTableMultiValuesWithColumns() { + TableValue tableValue = new TableValue(); + tableValue.setDatabase("test"); + tableValue.setName("weather"); + // columns + List columns = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + FieldMeta field = new FieldMeta(); + field.setName("f" + (i + 1)); + columns.add(field); + } + tableValue.setColumns(columns); + // values + List values = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + List fields = new ArrayList<>(); + for (int j = 0; j < 3; j++) { + FieldValue field = new FieldValue(); + field.setValue(j); + fields.add(field); + } + values.add(new RowValue(fields)); + } + tableValue.setValues(values); + + String sql = SqlSpeller.insertOneTableMultiValuesWithColumns(tableValue); + System.out.println(sql); + } + + @Test + public void insertMultiTableMultiValuesWithColumns() { + List tables = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + TableValue table = new TableValue(); + table.setDatabase("test"); + table.setName("t" + (i + 1)); + // columns + List columns = new ArrayList<>(); + for (int j = 0; j < 3; j++) { + FieldMeta field = new FieldMeta(); + field.setName("f" + (j + 1)); + columns.add(field); + } + table.setColumns(columns); + // values + List values = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + List fields = new ArrayList<>(); + for (int k = 0; k < columns.size(); k++) { + FieldValue field = new FieldValue<>(); + field.setValue((k + 1) * 10); + fields.add(field); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + tables.add(table); + } + + String sql = SqlSpeller.insertMultiTableMultiValuesWithColumns(tables); + System.out.println(sql); + } + + @Test + public void testInsertMultiTableMultiValues() { + List tables = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + TableValue table = new TableValue(); + table.setDatabase("test"); + table.setName("t" + (i + 1)); + List values = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + List fields = new ArrayList<>(); + for (int k = 0; k < 2; k++) { + FieldValue field = new FieldValue<>(); + field.setValue((k + 1) * 10); + fields.add(field); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + + tables.add(table); + } + + String sql = SqlSpeller.insertMultiTableMultiValues(tables); + System.out.println(sql); + } + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/TimeStampUtilTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/TimeStampUtilTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a4845677c55897c71791eba40104ea23de644f5c --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/TimeStampUtilTest.java @@ -0,0 +1,40 @@ +package com.taosdata.taosdemo.utils; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TimeStampUtilTest { + + @Test + public void datetimeToLong() { + final String startTime = "2005-01-01 00:00:00.000"; + long start = TimeStampUtil.datetimeToLong(startTime); + assertEquals(1104508800000l, start); + String dateTimeStr = TimeStampUtil.longToDatetime(start); + assertEquals("2005-01-01 00:00:00.000", dateTimeStr); + } + + @Test + public void longToDatetime() { + System.out.println(TimeStampUtil.longToDatetime(1293334499006l)); + + String datetime = TimeStampUtil.longToDatetime(1510000000000L); + assertEquals("2017-11-07 04:26:40.000", datetime); + long timestamp = TimeStampUtil.datetimeToLong(datetime); + assertEquals(1510000000000L, timestamp); + } + + @Test + public void range() { + long start = TimeStampUtil.datetimeToLong("2020-10-01 00:00:00.000"); + long timeGap = 1000; + long numOfRowsPerTable = 1000l * 3600l * 24l * 90l; + TimeStampUtil.TimeTuple timeTuple = TimeStampUtil.range(start, timeGap, numOfRowsPerTable); + System.out.println(TimeStampUtil.longToDatetime(timeTuple.start)); + System.out.println(TimeStampUtil.longToDatetime(timeTuple.end)); + System.out.println(timeTuple.timeGap); + + } + +} \ No newline at end of file diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 59bcb6eaff349e8b20fa13995d1a7297f9185a50..954fe468b1b8fc88ce93fa2474e2f69a33415e6f 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -1,8 +1,18 @@ PROJECT(TDengine) IF (TD_LINUX) + INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(demo apitest.c) + TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) +ENDIF () +IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(demo demo.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) ENDIF () diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index e08d667d6be42bef3dc4e058fab10ac68f0809e0..930a6075cae2767c0d50ca2e1574d6441430d1b3 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -467,7 +467,6 @@ int main(int argc, char *argv[]) { const char* passwd = "taosdata"; taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - taos_init(); TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { diff --git a/tests/examples/c/asyncdemo.c b/tests/examples/c/asyncdemo.c index c6cc89b31d6280c45ea30b33509eed5ebdf0dc08..16a14e96549c5cd66f747ae0c6c63f20c3e7fbfa 100644 --- a/tests/examples/c/asyncdemo.c +++ b/tests/examples/c/asyncdemo.c @@ -99,8 +99,6 @@ int main(int argc, char *argv[]) tableList = (STable *)malloc(size); memset(tableList, 0, size); - taos_init(); - taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) taos_error(taos); @@ -261,9 +259,6 @@ void taos_select_call_back(void *param, TAOS_RES *tres, int code) if (code == 0 && tres) { // asynchronous API to fetch a batch of records taos_fetch_rows_a(tres, taos_retrieve_call_back, pTable); - - // taos_fetch_row_a is a less efficient way to retrieve records since it call back app for every row - // taos_fetch_row_a(tres, taos_fetch_row_call_back, pTable); } else { printf("%s select failed, code:%d\n", pTable->name, code); diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 74a49288e9a1aa7081db45925bf52d6516e4801a..e074e6496607bf2046496c7b6766b6f3dedf9dbc 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -50,7 +50,7 @@ static void queryDB(TAOS *taos, char *command) { taos_free_result(pSql); } -void Test(char *qstr, const char *input, int i); +void Test(TAOS *taos, char *qstr, int i); int main(int argc, char *argv[]) { char qstr[1024]; @@ -61,23 +61,22 @@ int main(int argc, char *argv[]) { return 0; } - // init TAOS - taos_init(); + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/); + exit(1); + } for (int i = 0; i < 4000000; i++) { - Test(qstr, argv[1], i); + Test(taos, qstr, i); } + taos_close(taos); taos_cleanup(); } -void Test(char *qstr, const char *input, int index) { - TAOS *taos = taos_connect(input, "root", "taosdata", NULL, 0); +void Test(TAOS *taos, char *qstr, int index) { printf("==================test at %d\n================================", index); queryDB(taos, "drop database if exists demo"); queryDB(taos, "create database demo"); TAOS_RES *result; - if (taos == NULL) { - printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/); - exit(1); - } queryDB(taos, "use demo"); queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))"); @@ -85,23 +84,22 @@ void Test(char *qstr, const char *input, int index) { int i = 0; for (i = 0; i < 10; ++i) { - sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i * 1000), i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); // note: how do you wanna do if taos_query returns non-NULL // if (taos_query(taos, qstr)) { // printf("insert row: %i, reason:%s\n", i, taos_errstr(taos)); // } - TAOS_RES *result = taos_query(taos, qstr); - if (result) { - printf("insert row: %i\n", i); - } else { - printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); - taos_free_result(result); + TAOS_RES *result1 = taos_query(taos, qstr); + if (result1 == NULL || taos_errno(result1) != 0) { + printf("failed to insert row, reason:%s\n", taos_errstr(result1)); + taos_free_result(result1); exit(1); + } else { + printf("insert row: %i\n", i); } - taos_free_result(result); - + taos_free_result(result1); } printf("success to insert rows, total %d rows\n", i); @@ -131,6 +129,5 @@ void Test(char *qstr, const char *input, int index) { taos_free_result(result); printf("====demo end====\n\n"); - taos_close(taos); } diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c new file mode 100644 index 0000000000000000000000000000000000000000..de7c5989d1ecdaec578e4322ac49491587ad851b --- /dev/null +++ b/tests/examples/c/epoll.c @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// how to use to do a pressure-test upon eok +// tester: cat /dev/urandom | nc -c +// testee: ./debug/build/bin/epoll -l > /dev/null +// compare against: nc -l > /dev/null +// monitor and compare : glances + +#ifdef __APPLE__ +#include "eok.h" +#else // __APPLE__ +#include +#endif // __APPLE__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) + +#include "os.h" + +typedef struct ep_s ep_t; +struct ep_s { + int ep; + + pthread_mutex_t lock; + int sv[2]; // 0 for read, 1 for write; + pthread_t thread; + + volatile unsigned int stopping:1; + volatile unsigned int waiting:1; + volatile unsigned int wakenup:1; +}; + +static int ep_dummy = 0; + +static ep_t* ep_create(void); +static void ep_destroy(ep_t *ep); +static void* routine(void* arg); +static int open_listen(unsigned short port); + +typedef struct fde_s fde_t; +struct fde_s { + int skt; + void (*on_event)(ep_t *ep, struct epoll_event *events, fde_t *client); +}; + +static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client); +static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client); + +#define usage(arg0, fmt, ...) do { \ + if (fmt[0]) { \ + fprintf(stderr, "" fmt "\n", ##__VA_ARGS__); \ + } \ + fprintf(stderr, "usage:\n"); \ + fprintf(stderr, " %s -l : specify listenning port\n", arg0); \ +} while (0) + +int main(int argc, char *argv[]) { + char *prg = basename(argv[0]); + if (argc==1) { + usage(prg, ""); + return 0; + } + ep_t* ep = ep_create(); + A(ep, "failed"); + for (int i=1; i=argc) { + usage(prg, "expecting after -l, but got nothing"); + return 1; // confirmed potential leakage + } + arg = argv[i]; + int port = atoi(arg); + int skt = open_listen(port); + if (skt==-1) continue; + fde_t *client = (fde_t*)calloc(1, sizeof(*client)); + if (!client) { + E("out of memory"); + close(skt); + continue; + } + client->skt = skt; + client->on_event = listen_event; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = client; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + continue; + } + usage(prg, "unknown argument: [%s]", arg); + return 1; + } + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + while ((linelen = getline(&line, &linecap, stdin)) > 0) { + line[strlen(line)-1] = '\0'; + if (0==strcmp(line, "exit")) break; + if (0==strcmp(line, "quit")) break; + if (line==strstr(line, "close")) { + int fd = 0; + sscanf(line, "close %d", &fd); + if (fd<=2) { + fprintf(stderr, "fd [%d] invalid\n", fd); + continue; + } + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, fd, NULL), ""); + continue; + } + if (strlen(line)==0) continue; + fprintf(stderr, "unknown cmd:[%s]\n", line); + } + ep_destroy(ep); + D(""); + return 0; +} + +ep_t* ep_create(void) { + ep_t *ep = (ep_t*)calloc(1, sizeof(*ep)); + A(ep, "out of memory"); + A(-1!=(ep->ep = epoll_create(1)), ""); + ep->sv[0] = -1; + ep->sv[1] = -1; + A(0==socketpair(AF_LOCAL, SOCK_STREAM, 0, ep->sv), ""); + A(0==pthread_mutex_init(&ep->lock, NULL), ""); + A(0==pthread_mutex_lock(&ep->lock), ""); + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &ep_dummy; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, ep->sv[0], &ev), ""); + A(0==pthread_create(&ep->thread, NULL, routine, ep), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + return ep; +} + +static void ep_destroy(ep_t *ep) { + A(ep, "invalid argument"); + ep->stopping = 1; + A(1==send(ep->sv[1], "1", 1, 0), ""); + A(0==pthread_join(ep->thread, NULL), ""); + A(0==pthread_mutex_destroy(&ep->lock), ""); + A(0==close(ep->sv[0]), ""); + A(0==close(ep->sv[1]), ""); + A(0==close(ep->ep), ""); + free(ep); +} + +static void* routine(void* arg) { + A(arg, "invalid argument"); + ep_t *ep = (ep_t*)arg; + + while (!ep->stopping) { + struct epoll_event evs[10]; + memset(evs, 0, sizeof(evs)); + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==0, "internal logic error"); + ep->waiting = 1; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1); + A(r>0, "indefinite epoll_wait shall not timeout:[%d]", r); + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==1, "internal logic error"); + ep->waiting = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + for (int i=0; idata.ptr == &ep_dummy) { + char c = '\0'; + A(1==recv(ep->sv[0], &c, 1, 0), "internal logic error"); + A(0==pthread_mutex_lock(&ep->lock), ""); + ep->wakenup = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + continue; + } + A(ev->data.ptr, "internal logic error"); + fde_t *client = (fde_t*)ev->data.ptr; + client->on_event(ep, ev, client); + continue; + } + } + return NULL; +} + +static int open_listen(unsigned short port) { + int r = 0; + int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (skt==-1) { + E("socket() failed"); + return -1; + } + do { + struct sockaddr_in si = {0}; + si.sin_family = AF_INET; + si.sin_addr.s_addr = inet_addr("0.0.0.0"); + si.sin_port = htons(port); + r = bind(skt, (struct sockaddr*)&si, sizeof(si)); + if (r) { + E("bind(%u) failed", port); + break; + } + r = listen(skt, 100); + if (r) { + E("listen() failed"); + break; + } + memset(&si, 0, sizeof(si)); + socklen_t len = sizeof(si); + r = getsockname(skt, (struct sockaddr *)&si, &len); + if (r) { + E("getsockname() failed"); + } + A(len==sizeof(si), "internal logic error"); + D("listenning at: %d", ntohs(si.sin_port)); + return skt; + } while (0); + close(skt); + return -1; +} + +static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { + A(ev->events & EPOLLIN, "internal logic error"); + struct sockaddr_in si = {0}; + socklen_t silen = sizeof(si); + int skt = accept(client->skt, (struct sockaddr*)&si, &silen); + A(skt!=-1, "internal logic error"); + fde_t *server = (fde_t*)calloc(1, sizeof(*server)); + if (!server) { + close(skt); + return; + } + server->skt = skt; + server->on_event = null_event; + struct epoll_event ee = {0}; + ee.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ee.data.ptr = server; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ee), ""); +} + +static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { + if (ev->events & EPOLLIN) { + char buf[8192]; + int n = recv(client->skt, buf, sizeof(buf), 0); + A(n>=0 && n<=sizeof(buf), "internal logic error:[%d]", n); + A(n==fwrite(buf, 1, n, stdout), "internal logic error"); + } + if (ev->events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + A(0==pthread_mutex_lock(&ep->lock), ""); + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, client->skt, NULL), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + close(client->skt); + client->skt = -1; + client->on_event = NULL; + free(client); + } +} + diff --git a/tests/examples/c/makefile b/tests/examples/c/makefile index 7293a22c11d7c6049c220ca0951b5969fa344114..b06fe551dbde4b16e3ea197ca4c2bc1711ef63bb 100644 --- a/tests/examples/c/makefile +++ b/tests/examples/c/makefile @@ -6,7 +6,7 @@ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -msse4.2 -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 all: $(TARGET) diff --git a/tests/examples/c/prepare.c b/tests/examples/c/prepare.c index 7a70b744ee8561459318aa456160a54b8c6270a8..13d71beea6156aa32677c20ccc2d222d28826b24 100644 --- a/tests/examples/c/prepare.c +++ b/tests/examples/c/prepare.c @@ -22,9 +22,6 @@ int main(int argc, char *argv[]) return 0; } - // init TAOS - taos_init(); - taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); diff --git a/tests/examples/c/stream.c b/tests/examples/c/stream.c index 060f5b84ff276579019d3278552e424b2a4198e9..30a790f061cd8ef2b870a371c2cadfb0e2a413c1 100644 --- a/tests/examples/c/stream.c +++ b/tests/examples/c/stream.c @@ -54,9 +54,6 @@ int main(int argc, char *argv[]) exit(0); } - // init TAOS - taos_init(); - strcpy(db_name, argv[2]); strcpy(tbl_name, argv[3]); diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c index cdd8ddaf7f6c4d2e5088ef36cc00ad77a0c8ebc9..1d3533fa5ee89f7425a9b0563e738b4e0703e6c8 100644 --- a/tests/examples/c/subscribe.c +++ b/tests/examples/c/subscribe.c @@ -216,9 +216,6 @@ int main(int argc, char *argv[]) { } } - // init TAOS - taos_init(); - TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); diff --git a/tests/examples/lua/OpenResty/conf/nginx.conf b/tests/examples/lua/OpenResty/conf/nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..2f838c21fccd99ead5641fd7eea1e55b49851fa2 --- /dev/null +++ b/tests/examples/lua/OpenResty/conf/nginx.conf @@ -0,0 +1,21 @@ +worker_processes 1; +user root; +error_log logs/error.log; +events { + worker_connections 1024; +} + +http { + lua_package_path '$prefix/lua/?.lua;$prefix/rest/?.lua;/blah/?.lua;;'; + lua_package_cpath "$prefix/so/?.so;;"; + lua_code_cache off; + server { + listen 7000; + server_name restapi; + charset utf-8; + lua_need_request_body on; + location ~ ^/api/([-_a-zA-Z0-9/]+) { + content_by_lua_file rest/$1.lua; + } + } +} diff --git a/tests/examples/lua/OpenResty/logs/.gitignore b/tests/examples/lua/OpenResty/logs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..ad8530e1c3e3c88fcce3b50abf7cc006333f5522 --- /dev/null +++ b/tests/examples/lua/OpenResty/logs/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/tests/examples/lua/OpenResty/rest/test.lua b/tests/examples/lua/OpenResty/rest/test.lua new file mode 100644 index 0000000000000000000000000000000000000000..179950cbe7cc294cd53a538baecefda28fe30bcc --- /dev/null +++ b/tests/examples/lua/OpenResty/rest/test.lua @@ -0,0 +1,83 @@ +local driver = require "luaconnector51" +local cjson = require "cjson" +ngx.say("start time:"..os.time()) + + +local config = { + host = "127.0.0.1", + port = 6030, + database = "", + user = "root", + password = "taosdata", + max_packet_size = 1024 * 1024 +} + +local conn +local res = driver.connect(config) +if res.code ~=0 then + ngx.say("connect--- failed: "..res.error) + return +else + conn = res.conn + ngx.say("connect--- pass.") +end + +local res = driver.query(conn,"drop database if exists nginx") +if res.code ~=0 then + ngx.say("drop db--- failed: "..res.error) +else + ngx.say("drop db--- pass.") +end +res = driver.query(conn,"create database nginx") +if res.code ~=0 then + ngx.say("create db--- failed: "..res.error) + +else + ngx.say("create db--- pass.") +end + +res = driver.query(conn,"use nginx") +if res.code ~=0 then + ngx.say("select db--- failed: "..res.error) +else + ngx.say("select db--- pass.") +end + +res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") +if res.code ~=0 then + ngx.say("create table---failed: "..res.error) + +else + ngx.say("create table--- pass.") +end + +res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") +if res.code ~=0 then + ngx.say("insert records failed: "..res.error) + return +else + if(res.affected == 3) then + ngx.say("insert records--- pass") + else + ngx.say("insert records---failed: expect 3 affected records, actually affected "..res.affected) + end +end + +res = driver.query(conn,"select * from m1") + +if res.code ~=0 then + ngx.say("select failed: "..res.error) + return +else + ngx.say(cjson.encode(res)) + if (#(res.item) == 3) then + ngx.say("select--- pass") + else + ngx.say("select--- failed: expect 3 affected records, actually received "..#(res.item)) + end + +end +driver.close(conn) +ngx.say("end time:"..os.time()) +--ngx.log(ngx.ERR,"in test file.") + diff --git a/tests/examples/lua/OpenResty/so/luaconnector51.so b/tests/examples/lua/OpenResty/so/luaconnector51.so new file mode 100755 index 0000000000000000000000000000000000000000..442de6e39f909e1aeb869988722b84795c048855 Binary files /dev/null and b/tests/examples/lua/OpenResty/so/luaconnector51.so differ diff --git a/tests/examples/lua/README.md b/tests/examples/lua/README.md index 8efc52b35c96d7a45c2a26c4a6b9e3c2e66db5cc..dd9c9d07874e455329e43c7f77e806eb3634622c 100644 --- a/tests/examples/lua/README.md +++ b/tests/examples/lua/README.md @@ -2,13 +2,13 @@ It's a Lua implementation for [TDengine](https://github.com/taosdata/TDengine), an open-sourced big data platform designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. You may need to install Lua5.3 . -## Dependencies +## Lua Dependencies - Lua: ``` https://www.lua.org/ ``` -## Run with Sample +## Run with Lua Sample Build driver lib: ``` @@ -18,3 +18,26 @@ Run lua sample: ``` lua test.lua ``` + +## OpenResty Dependencies +- OpenResty: +``` +http://openresty.org +``` +## Run with OpenResty Sample +**This section demonstrates how to get binary file for connector. To be convenient for trial, an connector has been put into OpenResty work directory. +Because of difference on C API between Lua5.3 and Lua5.1, the files needed by connector for OpenResty are stored in local source directory and configured in script build.sh.** + +Build driver lib: +``` +cd lua51 +./build.sh +``` +Run OpenResty sample: +``` +cd .. +cd OpenResty +sudo openresty -p . +curl http://127.0.0.1:7000/api/test +``` + diff --git a/tests/examples/lua/lua51/build.sh b/tests/examples/lua/lua51/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..da2981bf7d748a7a42782ad34c0db5b7c666c437 --- /dev/null +++ b/tests/examples/lua/lua51/build.sh @@ -0,0 +1,2 @@ +gcc lua_connector51.c -fPIC -shared -o luaconnector51.so -Wall -ltaos + diff --git a/tests/examples/lua/lua51/lauxlib.h b/tests/examples/lua/lua51/lauxlib.h new file mode 100644 index 0000000000000000000000000000000000000000..a44f0272b3ab0745b0ce798d377b40d2cce113cd --- /dev/null +++ b/tests/examples/lua/lua51/lauxlib.h @@ -0,0 +1,161 @@ +/* +** $Id: lauxlib.h,v 1.88.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions for building Lua libraries +** See Copyright Notice in lua.h +*/ + + +#ifndef lauxlib_h +#define lauxlib_h + + +#include +#include + +#include "lua.h" + + +/* extra error code for `luaL_load' */ +#define LUA_ERRFILE (LUA_ERRERR+1) + +typedef struct luaL_Reg { + const char *name; + lua_CFunction func; +} luaL_Reg; + +LUALIB_API void (luaL_openlib) (lua_State *L, const char *libname, + const luaL_Reg *l, int nup); +LUALIB_API void (luaL_register) (lua_State *L, const char *libname, + const luaL_Reg *l); +LUALIB_API int (luaL_getmetafield) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_callmeta) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_typerror) (lua_State *L, int narg, const char *tname); +LUALIB_API int (luaL_argerror) (lua_State *L, int numarg, const char *extramsg); +LUALIB_API const char *(luaL_checklstring) (lua_State *L, int numArg, + size_t *l); +LUALIB_API const char *(luaL_optlstring) (lua_State *L, int numArg, + const char *def, size_t *l); +LUALIB_API lua_Number (luaL_checknumber) (lua_State *L, int numArg); +LUALIB_API lua_Number (luaL_optnumber) (lua_State *L, int nArg, lua_Number def); + +LUALIB_API lua_Integer (luaL_checkinteger) (lua_State *L, int numArg); +LUALIB_API lua_Integer (luaL_optinteger) (lua_State *L, int nArg, + lua_Integer def); + +LUALIB_API void (luaL_checkstack) (lua_State *L, int sz, const char *msg); +LUALIB_API void (luaL_checktype) (lua_State *L, int narg, int t); +LUALIB_API void (luaL_checkany) (lua_State *L, int narg); + +LUALIB_API int (luaL_newmetatable) (lua_State *L, const char *tname); +LUALIB_API void *(luaL_checkudata) (lua_State *L, int ud, const char *tname); + +LUALIB_API void (luaL_where) (lua_State *L, int lvl); +LUALIB_API int (luaL_error) (lua_State *L, const char *fmt, ...); + +LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def, + const char *const lst[]); + +/* pre-defined references */ +#define LUA_NOREF (-2) +#define LUA_REFNIL (-1) + +LUALIB_API int (luaL_ref) (lua_State *L, int t); +LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref); + +LUALIB_API int (luaL_loadfile) (lua_State *L, const char *filename); +LUALIB_API int (luaL_loadbuffer) (lua_State *L, const char *buff, size_t sz, + const char *name); +LUALIB_API int (luaL_loadstring) (lua_State *L, const char *s); + +LUALIB_API lua_State *(luaL_newstate) (void); + + +LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s, const char *p, + const char *r); + +LUALIB_API const char *(luaL_findtable) (lua_State *L, int idx, + const char *fname, int szhint); + +/* From Lua 5.2. */ +LUALIB_API int luaL_fileresult(lua_State *L, int stat, const char *fname); +LUALIB_API int luaL_execresult(lua_State *L, int stat); +LUALIB_API int (luaL_loadfilex) (lua_State *L, const char *filename, + const char *mode); +LUALIB_API int (luaL_loadbufferx) (lua_State *L, const char *buff, size_t sz, + const char *name, const char *mode); +LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, + int level); +LUALIB_API void (luaL_setfuncs) (lua_State *L, const luaL_Reg *l, int nup); +LUALIB_API void (luaL_pushmodule) (lua_State *L, const char *modname, + int sizehint); +LUALIB_API void *(luaL_testudata) (lua_State *L, int ud, const char *tname); +LUALIB_API void (luaL_setmetatable) (lua_State *L, const char *tname); + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define luaL_argcheck(L, cond,numarg,extramsg) \ + ((void)((cond) || luaL_argerror(L, (numarg), (extramsg)))) +#define luaL_checkstring(L,n) (luaL_checklstring(L, (n), NULL)) +#define luaL_optstring(L,n,d) (luaL_optlstring(L, (n), (d), NULL)) +#define luaL_checkint(L,n) ((int)luaL_checkinteger(L, (n))) +#define luaL_optint(L,n,d) ((int)luaL_optinteger(L, (n), (d))) +#define luaL_checklong(L,n) ((long)luaL_checkinteger(L, (n))) +#define luaL_optlong(L,n,d) ((long)luaL_optinteger(L, (n), (d))) + +#define luaL_typename(L,i) lua_typename(L, lua_type(L,(i))) + +#define luaL_dofile(L, fn) \ + (luaL_loadfile(L, fn) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_dostring(L, s) \ + (luaL_loadstring(L, s) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_getmetatable(L,n) (lua_getfield(L, LUA_REGISTRYINDEX, (n))) + +#define luaL_opt(L,f,n,d) (lua_isnoneornil(L,(n)) ? (d) : f(L,(n))) + +/* From Lua 5.2. */ +#define luaL_newlibtable(L, l) \ + lua_createtable(L, 0, sizeof(l)/sizeof((l)[0]) - 1) +#define luaL_newlib(L, l) (luaL_newlibtable(L, l), luaL_setfuncs(L, l, 0)) + +/* +** {====================================================== +** Generic Buffer manipulation +** ======================================================= +*/ + + + +typedef struct luaL_Buffer { + char *p; /* current position in buffer */ + int lvl; /* number of strings in the stack (level) */ + lua_State *L; + char buffer[LUAL_BUFFERSIZE]; +} luaL_Buffer; + +#define luaL_addchar(B,c) \ + ((void)((B)->p < ((B)->buffer+LUAL_BUFFERSIZE) || luaL_prepbuffer(B)), \ + (*(B)->p++ = (char)(c))) + +/* compatibility only */ +#define luaL_putchar(B,c) luaL_addchar(B,c) + +#define luaL_addsize(B,n) ((B)->p += (n)) + +LUALIB_API void (luaL_buffinit) (lua_State *L, luaL_Buffer *B); +LUALIB_API char *(luaL_prepbuffer) (luaL_Buffer *B); +LUALIB_API void (luaL_addlstring) (luaL_Buffer *B, const char *s, size_t l); +LUALIB_API void (luaL_addstring) (luaL_Buffer *B, const char *s); +LUALIB_API void (luaL_addvalue) (luaL_Buffer *B); +LUALIB_API void (luaL_pushresult) (luaL_Buffer *B); + + +/* }====================================================== */ + +#endif diff --git a/tests/examples/lua/lua51/lua.h b/tests/examples/lua/lua51/lua.h new file mode 100644 index 0000000000000000000000000000000000000000..9dcafd690655868115ce53dff26599f5845b12c5 --- /dev/null +++ b/tests/examples/lua/lua51/lua.h @@ -0,0 +1,404 @@ +/* +** $Id: lua.h,v 1.218.1.5 2008/08/06 13:30:12 roberto Exp $ +** Lua - An Extensible Extension Language +** Lua.org, PUC-Rio, Brazil (http://www.lua.org) +** See Copyright Notice at the end of this file +*/ + + +#ifndef lua_h +#define lua_h + +#include +#include + + +#include "luaconf.h" + + +#define LUA_VERSION "Lua 5.1" +#define LUA_RELEASE "Lua 5.1.4" +#define LUA_VERSION_NUM 501 +#define LUA_COPYRIGHT "Copyright (C) 1994-2008 Lua.org, PUC-Rio" +#define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo & W. Celes" + + +/* mark for precompiled code (`Lua') */ +#define LUA_SIGNATURE "\033Lua" + +/* option for multiple returns in `lua_pcall' and `lua_call' */ +#define LUA_MULTRET (-1) + + +/* +** pseudo-indices +*/ +#define LUA_REGISTRYINDEX (-10000) +#define LUA_ENVIRONINDEX (-10001) +#define LUA_GLOBALSINDEX (-10002) +#define lua_upvalueindex(i) (LUA_GLOBALSINDEX-(i)) + + +/* thread status */ +#define LUA_OK 0 +#define LUA_YIELD 1 +#define LUA_ERRRUN 2 +#define LUA_ERRSYNTAX 3 +#define LUA_ERRMEM 4 +#define LUA_ERRERR 5 + + +typedef struct lua_State lua_State; + +typedef int (*lua_CFunction) (lua_State *L); + + +/* +** functions that read/write blocks when loading/dumping Lua chunks +*/ +typedef const char * (*lua_Reader) (lua_State *L, void *ud, size_t *sz); + +typedef int (*lua_Writer) (lua_State *L, const void* p, size_t sz, void* ud); + + +/* +** prototype for memory-allocation functions +*/ +typedef void * (*lua_Alloc) (void *ud, void *ptr, size_t osize, size_t nsize); + + +/* +** basic types +*/ +#define LUA_TNONE (-1) + +#define LUA_TNIL 0 +#define LUA_TBOOLEAN 1 +#define LUA_TLIGHTUSERDATA 2 +#define LUA_TNUMBER 3 +#define LUA_TSTRING 4 +#define LUA_TTABLE 5 +#define LUA_TFUNCTION 6 +#define LUA_TUSERDATA 7 +#define LUA_TTHREAD 8 + + + +/* minimum Lua stack available to a C function */ +#define LUA_MINSTACK 20 + + +/* +** generic extra include file +*/ +#if defined(LUA_USER_H) +#include LUA_USER_H +#endif + + +/* type of numbers in Lua */ +typedef LUA_NUMBER lua_Number; + + +/* type for integer functions */ +typedef LUA_INTEGER lua_Integer; + + + +/* +** state manipulation +*/ +LUA_API lua_State *(lua_newstate) (lua_Alloc f, void *ud); +LUA_API void (lua_close) (lua_State *L); +LUA_API lua_State *(lua_newthread) (lua_State *L); + +LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf); + + +/* +** basic stack manipulation +*/ +LUA_API int (lua_gettop) (lua_State *L); +LUA_API void (lua_settop) (lua_State *L, int idx); +LUA_API void (lua_pushvalue) (lua_State *L, int idx); +LUA_API void (lua_remove) (lua_State *L, int idx); +LUA_API void (lua_insert) (lua_State *L, int idx); +LUA_API void (lua_replace) (lua_State *L, int idx); +LUA_API int (lua_checkstack) (lua_State *L, int sz); + +LUA_API void (lua_xmove) (lua_State *from, lua_State *to, int n); + + +/* +** access functions (stack -> C) +*/ + +LUA_API int (lua_isnumber) (lua_State *L, int idx); +LUA_API int (lua_isstring) (lua_State *L, int idx); +LUA_API int (lua_iscfunction) (lua_State *L, int idx); +LUA_API int (lua_isuserdata) (lua_State *L, int idx); +LUA_API int (lua_type) (lua_State *L, int idx); +LUA_API const char *(lua_typename) (lua_State *L, int tp); + +LUA_API int (lua_equal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_rawequal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_lessthan) (lua_State *L, int idx1, int idx2); + +LUA_API lua_Number (lua_tonumber) (lua_State *L, int idx); +LUA_API lua_Integer (lua_tointeger) (lua_State *L, int idx); +LUA_API int (lua_toboolean) (lua_State *L, int idx); +LUA_API const char *(lua_tolstring) (lua_State *L, int idx, size_t *len); +LUA_API size_t (lua_objlen) (lua_State *L, int idx); +LUA_API lua_CFunction (lua_tocfunction) (lua_State *L, int idx); +LUA_API void *(lua_touserdata) (lua_State *L, int idx); +LUA_API lua_State *(lua_tothread) (lua_State *L, int idx); +LUA_API const void *(lua_topointer) (lua_State *L, int idx); + + +/* +** push functions (C -> stack) +*/ +LUA_API void (lua_pushnil) (lua_State *L); +LUA_API void (lua_pushnumber) (lua_State *L, lua_Number n); +LUA_API void (lua_pushinteger) (lua_State *L, lua_Integer n); +LUA_API void (lua_pushlstring) (lua_State *L, const char *s, size_t l); +LUA_API void (lua_pushstring) (lua_State *L, const char *s); +LUA_API const char *(lua_pushvfstring) (lua_State *L, const char *fmt, + va_list argp); +LUA_API const char *(lua_pushfstring) (lua_State *L, const char *fmt, ...); +LUA_API void (lua_pushcclosure) (lua_State *L, lua_CFunction fn, int n); +LUA_API void (lua_pushboolean) (lua_State *L, int b); +LUA_API void (lua_pushlightuserdata) (lua_State *L, void *p); +LUA_API int (lua_pushthread) (lua_State *L); + + +/* +** get functions (Lua -> stack) +*/ +LUA_API void (lua_gettable) (lua_State *L, int idx); +LUA_API void (lua_getfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawget) (lua_State *L, int idx); +LUA_API void (lua_rawgeti) (lua_State *L, int idx, int n); +LUA_API void (lua_createtable) (lua_State *L, int narr, int nrec); +LUA_API void *(lua_newuserdata) (lua_State *L, size_t sz); +LUA_API int (lua_getmetatable) (lua_State *L, int objindex); +LUA_API void (lua_getfenv) (lua_State *L, int idx); + + +/* +** set functions (stack -> Lua) +*/ +LUA_API void (lua_settable) (lua_State *L, int idx); +LUA_API void (lua_setfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawset) (lua_State *L, int idx); +LUA_API void (lua_rawseti) (lua_State *L, int idx, int n); +LUA_API int (lua_setmetatable) (lua_State *L, int objindex); +LUA_API int (lua_setfenv) (lua_State *L, int idx); + + +/* +** `load' and `call' functions (load and run Lua code) +*/ +LUA_API void (lua_call) (lua_State *L, int nargs, int nresults); +LUA_API int (lua_pcall) (lua_State *L, int nargs, int nresults, int errfunc); +LUA_API int (lua_cpcall) (lua_State *L, lua_CFunction func, void *ud); +LUA_API int (lua_load) (lua_State *L, lua_Reader reader, void *dt, + const char *chunkname); + +LUA_API int (lua_dump) (lua_State *L, lua_Writer writer, void *data); + + +/* +** coroutine functions +*/ +LUA_API int (lua_yield) (lua_State *L, int nresults); +LUA_API int (lua_resume) (lua_State *L, int narg); +LUA_API int (lua_status) (lua_State *L); + +/* +** garbage-collection function and options +*/ + +#define LUA_GCSTOP 0 +#define LUA_GCRESTART 1 +#define LUA_GCCOLLECT 2 +#define LUA_GCCOUNT 3 +#define LUA_GCCOUNTB 4 +#define LUA_GCSTEP 5 +#define LUA_GCSETPAUSE 6 +#define LUA_GCSETSTEPMUL 7 +#define LUA_GCISRUNNING 9 + +LUA_API int (lua_gc) (lua_State *L, int what, int data); + + +/* +** miscellaneous functions +*/ + +LUA_API int (lua_error) (lua_State *L); + +LUA_API int (lua_next) (lua_State *L, int idx); + +LUA_API void (lua_concat) (lua_State *L, int n); + +LUA_API lua_Alloc (lua_getallocf) (lua_State *L, void **ud); +LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud); + +LUA_API void lua_setexdata(lua_State *L, void *exdata); +LUA_API void *lua_getexdata(lua_State *L); + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define lua_pop(L,n) lua_settop(L, -(n)-1) + +#define lua_newtable(L) lua_createtable(L, 0, 0) + +#define lua_register(L,n,f) (lua_pushcfunction(L, (f)), lua_setglobal(L, (n))) + +#define lua_pushcfunction(L,f) lua_pushcclosure(L, (f), 0) + +#define lua_strlen(L,i) lua_objlen(L, (i)) + +#define lua_isfunction(L,n) (lua_type(L, (n)) == LUA_TFUNCTION) +#define lua_istable(L,n) (lua_type(L, (n)) == LUA_TTABLE) +#define lua_islightuserdata(L,n) (lua_type(L, (n)) == LUA_TLIGHTUSERDATA) +#define lua_isnil(L,n) (lua_type(L, (n)) == LUA_TNIL) +#define lua_isboolean(L,n) (lua_type(L, (n)) == LUA_TBOOLEAN) +#define lua_isthread(L,n) (lua_type(L, (n)) == LUA_TTHREAD) +#define lua_isnone(L,n) (lua_type(L, (n)) == LUA_TNONE) +#define lua_isnoneornil(L, n) (lua_type(L, (n)) <= 0) + +#define lua_pushliteral(L, s) \ + lua_pushlstring(L, "" s, (sizeof(s)/sizeof(char))-1) + +#define lua_setglobal(L,s) lua_setfield(L, LUA_GLOBALSINDEX, (s)) +#define lua_getglobal(L,s) lua_getfield(L, LUA_GLOBALSINDEX, (s)) + +#define lua_tostring(L,i) lua_tolstring(L, (i), NULL) + + + +/* +** compatibility macros and functions +*/ + +#define lua_open() luaL_newstate() + +#define lua_getregistry(L) lua_pushvalue(L, LUA_REGISTRYINDEX) + +#define lua_getgccount(L) lua_gc(L, LUA_GCCOUNT, 0) + +#define lua_Chunkreader lua_Reader +#define lua_Chunkwriter lua_Writer + + +/* hack */ +LUA_API void lua_setlevel (lua_State *from, lua_State *to); + + +/* +** {====================================================================== +** Debug API +** ======================================================================= +*/ + + +/* +** Event codes +*/ +#define LUA_HOOKCALL 0 +#define LUA_HOOKRET 1 +#define LUA_HOOKLINE 2 +#define LUA_HOOKCOUNT 3 +#define LUA_HOOKTAILRET 4 + + +/* +** Event masks +*/ +#define LUA_MASKCALL (1 << LUA_HOOKCALL) +#define LUA_MASKRET (1 << LUA_HOOKRET) +#define LUA_MASKLINE (1 << LUA_HOOKLINE) +#define LUA_MASKCOUNT (1 << LUA_HOOKCOUNT) + +typedef struct lua_Debug lua_Debug; /* activation record */ + + +/* Functions to be called by the debuger in specific events */ +typedef void (*lua_Hook) (lua_State *L, lua_Debug *ar); + + +LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar); +LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar); +LUA_API const char *lua_getlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_getupvalue (lua_State *L, int funcindex, int n); +LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n); +LUA_API int lua_sethook (lua_State *L, lua_Hook func, int mask, int count); +LUA_API lua_Hook lua_gethook (lua_State *L); +LUA_API int lua_gethookmask (lua_State *L); +LUA_API int lua_gethookcount (lua_State *L); + +/* From Lua 5.2. */ +LUA_API void *lua_upvalueid (lua_State *L, int idx, int n); +LUA_API void lua_upvaluejoin (lua_State *L, int idx1, int n1, int idx2, int n2); +LUA_API int lua_loadx (lua_State *L, lua_Reader reader, void *dt, + const char *chunkname, const char *mode); +LUA_API const lua_Number *lua_version (lua_State *L); +LUA_API void lua_copy (lua_State *L, int fromidx, int toidx); +LUA_API lua_Number lua_tonumberx (lua_State *L, int idx, int *isnum); +LUA_API lua_Integer lua_tointegerx (lua_State *L, int idx, int *isnum); + +/* From Lua 5.3. */ +LUA_API int lua_isyieldable (lua_State *L); + + +struct lua_Debug { + int event; + const char *name; /* (n) */ + const char *namewhat; /* (n) `global', `local', `field', `method' */ + const char *what; /* (S) `Lua', `C', `main', `tail' */ + const char *source; /* (S) */ + int currentline; /* (l) */ + int nups; /* (u) number of upvalues */ + int linedefined; /* (S) */ + int lastlinedefined; /* (S) */ + char short_src[LUA_IDSIZE]; /* (S) */ + /* private part */ + int i_ci; /* active function */ +}; + +/* }====================================================================== */ + + +/****************************************************************************** +* Copyright (C) 1994-2008 Lua.org, PUC-Rio. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +******************************************************************************/ + + +#endif diff --git a/tests/examples/lua/lua51/lua_connector51.c b/tests/examples/lua/lua51/lua_connector51.c new file mode 100644 index 0000000000000000000000000000000000000000..9b932337febb204eada021ececa02bc59cf6d5db --- /dev/null +++ b/tests/examples/lua/lua51/lua_connector51.c @@ -0,0 +1,323 @@ +#include +#include +#include +#include +#include "lua.h" +#include "lauxlib.h" +#include "lualib.h" +#include + +struct cb_param{ + lua_State* state; + int callback; + void * stream; +}; + +static int l_connect(lua_State *L){ + TAOS * taos=NULL; + const char* host; + const char* database; + const char* user; + const char* password; + int port; + + luaL_checktype(L, 1, LUA_TTABLE); + + lua_getfield(L,-1,"host"); + if (lua_isstring(L,-1)){ + host = lua_tostring(L, -1); + // printf("host = %s\n", host); + } + + lua_getfield(L, 1, "port"); + if (lua_isnumber(L,-1)){ + port = lua_tonumber(L, -1); + //printf("port = %d\n", port); + } + + lua_getfield(L, 1, "database"); + if (lua_isstring(L, -1)){ + database = lua_tostring(L, -1); + //printf("database = %s\n", database); + } + + lua_getfield(L, 1, "user"); + if (lua_isstring(L, -1)){ + user = lua_tostring(L, -1); + //printf("user = %s\n", user); + } + + lua_getfield(L, 1, "password"); + if (lua_isstring(L, -1)){ + password = lua_tostring(L, -1); + //printf("password = %s\n", password); + } + + lua_settop(L,0); + + taos_init(); + + lua_newtable(L); + int table_index = lua_gettop(L); + + taos = taos_connect(host, user,password,database, port); + if (taos == NULL) { + printf("failed to connect server, reason:%s\n", taos_errstr(taos)); + + lua_pushinteger(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + lua_pushlightuserdata(L,NULL); + lua_setfield(L, table_index, "conn"); + }else{ + // printf("success to connect server\n"); + lua_pushinteger(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + lua_pushlightuserdata(L,taos); + lua_setfield(L, table_index, "conn"); + } + + return 1; +} + +static int l_query(lua_State *L){ + TAOS *taos= (TAOS*)lua_topointer(L,1); + const char* s = lua_tostring(L, 2); + TAOS_RES *result; + lua_newtable(L); + int table_index = lua_gettop(L); + + // printf("receive command:%s\r\n",s); + result = taos_query(taos, s); + int32_t code = taos_errno(result); + if( code != 0){ + printf("failed, reason:%s\n", taos_errstr(result)); + lua_pushinteger(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + + return 1; + + }else{ + //printf("success to query.\n"); + TAOS_ROW row; + int rows = 0; + int num_fields = taos_field_count(result); + const TAOS_FIELD *fields = taos_fetch_fields(result); + //char temp[256]; + + const int affectRows = taos_affected_rows(result); + // printf(" affect rows:%d\r\n", affectRows); + lua_pushinteger(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushinteger(L, affectRows); + lua_setfield(L, table_index, "affected"); + lua_newtable(L); + + while ((row = taos_fetch_row(result))) { + //printf("row index:%d\n",rows); + rows++; + + lua_pushnumber(L,rows); + lua_newtable(L); + + for (int i = 0; i < num_fields; ++i) { + if (row[i] == NULL) { + continue; + } + + lua_pushstring(L,fields[i].name); + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + lua_pushinteger(L,*((char *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + lua_pushinteger(L,*((short *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + lua_pushinteger(L,*((int *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: + lua_pushnumber(L,*((float *)row[i])); + break; + case TSDB_DATA_TYPE_DOUBLE: + lua_pushnumber(L,*((double *)row[i])); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + lua_pushstring(L,(char *)row[i]); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_BOOL: + lua_pushinteger(L,*((char *)row[i])); + break; + default: + lua_pushnil(L); + break; + } + + lua_settable(L,-3); + } + + lua_settable(L,-3); + } + taos_free_result(result); + } + + lua_setfield(L, table_index, "item"); + return 1; +} + +void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ + struct cb_param* p = (struct cb_param*) param; + TAOS_FIELD *fields = taos_fetch_fields(result); + int numFields = taos_num_fields(result); + + // printf("\nnumfields:%d\n", numFields); + //printf("\n\r-----------------------------------------------------------------------------------\n"); + + lua_State *L = p->state; + lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); + + lua_newtable(L); + + for (int i = 0; i < numFields; ++i) { + if (row[i] == NULL) { + continue; + } + + lua_pushstring(L,fields[i].name); + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + lua_pushinteger(L,*((char *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + lua_pushinteger(L,*((short *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + lua_pushinteger(L,*((int *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: + lua_pushnumber(L,*((float *)row[i])); + break; + case TSDB_DATA_TYPE_DOUBLE: + lua_pushnumber(L,*((double *)row[i])); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + lua_pushstring(L,(char *)row[i]); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_BOOL: + lua_pushinteger(L,*((char *)row[i])); + break; + default: + lua_pushnil(L); + break; + } + + lua_settable(L, -3); + } + + lua_call(L, 1, 0); + + // printf("-----------------------------------------------------------------------------------\n\r"); +} + +static int l_open_stream(lua_State *L){ + int r = luaL_ref(L, LUA_REGISTRYINDEX); + TAOS * taos = (TAOS*)lua_topointer(L,1); + const char * sqlstr = lua_tostring(L,2); + int stime = luaL_checknumber(L,3); + + lua_newtable(L); + int table_index = lua_gettop(L); + + struct cb_param *p = malloc(sizeof(struct cb_param)); + p->state = L; + p->callback=r; + // printf("r:%d, L:%d\n",r,L); + void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL); + if (s == NULL) { + printf("failed to open stream, reason:%s\n", taos_errstr(taos)); + free(p); + lua_pushnumber(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + lua_pushlightuserdata(L,NULL); + lua_setfield(L, table_index, "stream"); + }else{ + // printf("success to open stream\n"); + lua_pushnumber(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + p->stream = s; + lua_pushlightuserdata(L,p); + lua_setfield(L, table_index, "stream");//stream has different content in lua and c. + } + + return 1; +} + +static int l_close_stream(lua_State *L){ + //TODO:get stream and free cb_param + struct cb_param *p = lua_touserdata(L,1); + taos_close_stream(p->stream); + free(p); + return 0; +} + +static int l_close(lua_State *L){ + TAOS *taos= (TAOS*)lua_topointer(L,1); + lua_newtable(L); + int table_index = lua_gettop(L); + + if(taos == NULL){ + lua_pushnumber(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, "null pointer."); + lua_setfield(L, table_index, "error"); + }else{ + taos_close(taos); + lua_pushnumber(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, "done."); + lua_setfield(L, table_index, "error"); + } + return 1; +} + +static const struct luaL_Reg lib[] = { + {"connect", l_connect}, + {"query", l_query}, + {"close", l_close}, + {"open_stream", l_open_stream}, + {"close_stream", l_close_stream}, + {NULL, NULL} +}; + +extern int luaopen_luaconnector51(lua_State* L) +{ + // luaL_register(L, "luaconnector51", lib); + lua_newtable (L); + luaL_setfuncs(L,lib,0); + return 1; +} diff --git a/tests/examples/lua/lua51/luaconf.h b/tests/examples/lua/lua51/luaconf.h new file mode 100644 index 0000000000000000000000000000000000000000..c72893fd152e23c2a413c2da1d31be7b860b2a39 --- /dev/null +++ b/tests/examples/lua/lua51/luaconf.h @@ -0,0 +1,152 @@ +/* +** Configuration header. +** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h +*/ + +#ifndef luaconf_h +#define luaconf_h + +#ifndef WINVER +#define WINVER 0x0501 +#endif +#include +#include + +/* Default path for loading Lua and C modules with require(). */ +#if defined(_WIN32) +/* +** In Windows, any exclamation mark ('!') in the path is replaced by the +** path of the directory of the executable file of the current process. +*/ +#define LUA_LDIR "!\\lua\\" +#define LUA_CDIR "!\\" +#define LUA_PATH_DEFAULT \ + ".\\?.lua;" "!\\lualib\\?.lua;" LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" +#define LUA_CPATH_DEFAULT \ + ".\\?.dll;" "!\\lualib\\?.so;" LUA_CDIR"?.dll;" LUA_CDIR"loadall.dll" +#else +/* +** Note to distribution maintainers: do NOT patch the following lines! +** Please read ../doc/install.html#distro and pass PREFIX=/usr instead. +*/ +#ifndef LUA_MULTILIB +#define LUA_MULTILIB "lib" +#endif +#ifndef LUA_LMULTILIB +#define LUA_LMULTILIB "lib" +#endif +#define LUA_LROOT "/usr/local" +#define LUA_LUADIR "/lua/5.1/" +#define LUA_LJDIR "/luajit-2.1.0-beta3/" + +#ifdef LUA_ROOT +#define LUA_JROOT LUA_ROOT +#define LUA_RLDIR LUA_ROOT "/share" LUA_LUADIR +#define LUA_RCDIR LUA_ROOT "/" LUA_MULTILIB LUA_LUADIR +#define LUA_RLPATH ";" LUA_RLDIR "?.lua;" LUA_RLDIR "?/init.lua" +#define LUA_RCPATH ";" LUA_RCDIR "?.so" +#else +#define LUA_JROOT LUA_LROOT +#define LUA_RLPATH +#define LUA_RCPATH +#endif + +#define LUA_JPATH ";" LUA_JROOT "/share" LUA_LJDIR "?.lua" +#define LUA_LLDIR LUA_LROOT "/share" LUA_LUADIR +#define LUA_LCDIR LUA_LROOT "/" LUA_LMULTILIB LUA_LUADIR +#define LUA_LLPATH ";" LUA_LLDIR "?.lua;" LUA_LLDIR "?/init.lua" +#define LUA_LCPATH1 ";" LUA_LCDIR "?.so" +#define LUA_LCPATH2 ";" LUA_LCDIR "loadall.so" + +#define LUA_PATH_DEFAULT "./?.lua" LUA_JPATH LUA_LLPATH LUA_RLPATH +#define LUA_CPATH_DEFAULT "./?.so" LUA_LCPATH1 LUA_RCPATH LUA_LCPATH2 +#endif + +/* Environment variable names for path overrides and initialization code. */ +#define LUA_PATH "LUA_PATH" +#define LUA_CPATH "LUA_CPATH" +#define LUA_INIT "LUA_INIT" + +/* Special file system characters. */ +#if defined(_WIN32) +#define LUA_DIRSEP "\\" +#else +#define LUA_DIRSEP "/" +#endif +#define LUA_PATHSEP ";" +#define LUA_PATH_MARK "?" +#define LUA_EXECDIR "!" +#define LUA_IGMARK "-" +#define LUA_PATH_CONFIG \ + LUA_DIRSEP "\n" LUA_PATHSEP "\n" LUA_PATH_MARK "\n" \ + LUA_EXECDIR "\n" LUA_IGMARK "\n" + +/* Quoting in error messages. */ +#define LUA_QL(x) "'" x "'" +#define LUA_QS LUA_QL("%s") + +/* Various tunables. */ +#define LUAI_MAXSTACK 65500 /* Max. # of stack slots for a thread (<64K). */ +#define LUAI_MAXCSTACK 8000 /* Max. # of stack slots for a C func (<10K). */ +#define LUAI_GCPAUSE 200 /* Pause GC until memory is at 200%. */ +#define LUAI_GCMUL 200 /* Run GC at 200% of allocation speed. */ +#define LUA_MAXCAPTURES 32 /* Max. pattern captures. */ + +/* Configuration for the frontend (the luajit executable). */ +#if defined(luajit_c) +#define LUA_PROGNAME "luajit" /* Fallback frontend name. */ +#define LUA_PROMPT "> " /* Interactive prompt. */ +#define LUA_PROMPT2 ">> " /* Continuation prompt. */ +#define LUA_MAXINPUT 512 /* Max. input line length. */ +#endif + +/* Note: changing the following defines breaks the Lua 5.1 ABI. */ +#define LUA_INTEGER ptrdiff_t +#define LUA_IDSIZE 60 /* Size of lua_Debug.short_src. */ +/* +** Size of lauxlib and io.* on-stack buffers. Weird workaround to avoid using +** unreasonable amounts of stack space, but still retain ABI compatibility. +** Blame Lua for depending on BUFSIZ in the ABI, blame **** for wrecking it. +*/ +#define LUAL_BUFFERSIZE (BUFSIZ > 16384 ? 8192 : BUFSIZ) + +/* The following defines are here only for compatibility with luaconf.h +** from the standard Lua distribution. They must not be changed for LuaJIT. +*/ +#define LUA_NUMBER_DOUBLE +#define LUA_NUMBER double +#define LUAI_UACNUMBER double +#define LUA_NUMBER_SCAN "%lf" +#define LUA_NUMBER_FMT "%.14g" +#define lua_number2str(s, n) sprintf((s), LUA_NUMBER_FMT, (n)) +#define LUAI_MAXNUMBER2STR 32 +#define LUA_INTFRMLEN "l" +#define LUA_INTFRM_T long + +/* Linkage of public API functions. */ +#if defined(LUA_BUILD_AS_DLL) +#if defined(LUA_CORE) || defined(LUA_LIB) +#define LUA_API __declspec(dllexport) +#else +#define LUA_API __declspec(dllimport) +#endif +#else +#define LUA_API extern +#endif + +#define LUALIB_API LUA_API + +/* Support for internal assertions. */ +#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK) +#include +#endif +#ifdef LUA_USE_ASSERT +#define lua_assert(x) assert(x) +#endif +#ifdef LUA_USE_APICHECK +#define luai_apicheck(L, o) { (void)L; assert(o); } +#else +#define luai_apicheck(L, o) { (void)L; } +#endif + +#endif diff --git a/tests/examples/lua/lua51/luajit.h b/tests/examples/lua/lua51/luajit.h new file mode 100644 index 0000000000000000000000000000000000000000..ae14c4ffebdff746b22b2274abf33b5c9d601b26 --- /dev/null +++ b/tests/examples/lua/lua51/luajit.h @@ -0,0 +1,81 @@ +/* +** LuaJIT -- a Just-In-Time Compiler for Lua. http://luajit.org/ +** +** Copyright (C) 2005-2017 Mike Pall. All rights reserved. +** +** Permission is hereby granted, free of charge, to any person obtaining +** a copy of this software and associated documentation files (the +** "Software"), to deal in the Software without restriction, including +** without limitation the rights to use, copy, modify, merge, publish, +** distribute, sublicense, and/or sell copies of the Software, and to +** permit persons to whom the Software is furnished to do so, subject to +** the following conditions: +** +** The above copyright notice and this permission notice shall be +** included in all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +** +** [ MIT license: http://www.opensource.org/licenses/mit-license.php ] +*/ + +#ifndef _LUAJIT_H +#define _LUAJIT_H + +#include "lua.h" + +#define OPENRESTY_LUAJIT + +#define LUAJIT_VERSION "LuaJIT 2.1.0-beta3" +#define LUAJIT_VERSION_NUM 20100 /* Version 2.1.0 = 02.01.00. */ +#define LUAJIT_VERSION_SYM luaJIT_version_2_1_0_beta3 +#define LUAJIT_COPYRIGHT "Copyright (C) 2005-2017 Mike Pall" +#define LUAJIT_URL "http://luajit.org/" + +/* Modes for luaJIT_setmode. */ +#define LUAJIT_MODE_MASK 0x00ff + +enum { + LUAJIT_MODE_ENGINE, /* Set mode for whole JIT engine. */ + LUAJIT_MODE_DEBUG, /* Set debug mode (idx = level). */ + + LUAJIT_MODE_FUNC, /* Change mode for a function. */ + LUAJIT_MODE_ALLFUNC, /* Recurse into subroutine protos. */ + LUAJIT_MODE_ALLSUBFUNC, /* Change only the subroutines. */ + + LUAJIT_MODE_TRACE, /* Flush a compiled trace. */ + + LUAJIT_MODE_WRAPCFUNC = 0x10, /* Set wrapper mode for C function calls. */ + + LUAJIT_MODE_MAX +}; + +/* Flags or'ed in to the mode. */ +#define LUAJIT_MODE_OFF 0x0000 /* Turn feature off. */ +#define LUAJIT_MODE_ON 0x0100 /* Turn feature on. */ +#define LUAJIT_MODE_FLUSH 0x0200 /* Flush JIT-compiled code. */ + +/* LuaJIT public C API. */ + +/* Control the JIT engine. */ +LUA_API int luaJIT_setmode(lua_State *L, int idx, int mode); + +/* Low-overhead profiling API. */ +typedef void (*luaJIT_profile_callback)(void *data, lua_State *L, + int samples, int vmstate); +LUA_API void luaJIT_profile_start(lua_State *L, const char *mode, + luaJIT_profile_callback cb, void *data); +LUA_API void luaJIT_profile_stop(lua_State *L); +LUA_API const char *luaJIT_profile_dumpstack(lua_State *L, const char *fmt, + int depth, size_t *len); + +/* Enforce (dynamic) linker error for version mismatches. Call from main. */ +LUA_API void LUAJIT_VERSION_SYM(void); + +#endif diff --git a/tests/examples/lua/lua51/lualib.h b/tests/examples/lua/lua51/lualib.h new file mode 100644 index 0000000000000000000000000000000000000000..6aceabe59218d1863e6493bb77b9bed18bc18bdc --- /dev/null +++ b/tests/examples/lua/lua51/lualib.h @@ -0,0 +1,44 @@ +/* +** Standard library header. +** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h +*/ + +#ifndef _LUALIB_H +#define _LUALIB_H + +#include "lua.h" + +#define LUA_FILEHANDLE "FILE*" + +#define LUA_COLIBNAME "coroutine" +#define LUA_MATHLIBNAME "math" +#define LUA_STRLIBNAME "string" +#define LUA_TABLIBNAME "table" +#define LUA_IOLIBNAME "io" +#define LUA_OSLIBNAME "os" +#define LUA_LOADLIBNAME "package" +#define LUA_DBLIBNAME "debug" +#define LUA_BITLIBNAME "bit" +#define LUA_JITLIBNAME "jit" +#define LUA_FFILIBNAME "ffi" +#define LUA_THRLIBNAME "thread" + +LUALIB_API int luaopen_base(lua_State *L); +LUALIB_API int luaopen_math(lua_State *L); +LUALIB_API int luaopen_string(lua_State *L); +LUALIB_API int luaopen_table(lua_State *L); +LUALIB_API int luaopen_io(lua_State *L); +LUALIB_API int luaopen_os(lua_State *L); +LUALIB_API int luaopen_package(lua_State *L); +LUALIB_API int luaopen_debug(lua_State *L); +LUALIB_API int luaopen_bit(lua_State *L); +LUALIB_API int luaopen_jit(lua_State *L); +LUALIB_API int luaopen_ffi(lua_State *L); + +LUALIB_API void luaL_openlibs(lua_State *L); + +#ifndef lua_assert +#define lua_assert(x) ((void)0) +#endif + +#endif diff --git a/tests/examples/lua/lua_connector.c b/tests/examples/lua/lua_connector.c index 143f16a799244d4b77f3d01c4d2189fa6519b0d9..920d2cdc35c51c833a4d89448ec7e643f555dbc2 100644 --- a/tests/examples/lua/lua_connector.c +++ b/tests/examples/lua/lua_connector.c @@ -13,17 +13,49 @@ struct cb_param{ void * stream; }; +static int l_connect(lua_State *L){ + TAOS * taos=NULL; + const char* host; + const char* database; + const char* user; + const char* password; + int port; + + luaL_checktype(L, 1, LUA_TTABLE); + + lua_getfield(L,-1,"host"); + if (lua_isstring(L,-1)){ + host = lua_tostring(L, -1); + // printf("host = %s\n", host); + } + + lua_getfield(L, 1, "port"); + if (lua_isinteger(L,-1)){ + port = lua_tointeger(L, -1); + //printf("port = %d\n", port); + } + + lua_getfield(L, 1, "database"); + if (lua_isstring(L, -1)){ + database = lua_tostring(L, -1); + //printf("database = %s\n", database); + } + + lua_getfield(L, 1, "user"); + if (lua_isstring(L, -1)){ + user = lua_tostring(L, -1); + //printf("user = %s\n", user); + } + + lua_getfield(L, 1, "password"); + if (lua_isstring(L, -1)){ + password = lua_tostring(L, -1); + //printf("password = %s\n", password); + } + lua_settop(L,0); -static int l_connect(lua_State *L){ - TAOS * taos; - char *host = lua_tostring(L, 1); - char *user = lua_tostring(L, 2); - char *password = lua_tostring(L, 3); - char *database = lua_tostring(L, 4); - int port =luaL_checknumber(L, 5); taos_init(); - lua_newtable(L); int table_index = lua_gettop(L); @@ -31,38 +63,38 @@ static int l_connect(lua_State *L){ if (taos == NULL) { printf("failed to connect server, reason:%s\n", taos_errstr(taos)); - lua_pushnumber(L, -1); + lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); lua_setfield(L, table_index, "error"); lua_pushlightuserdata(L,NULL); lua_setfield(L, table_index, "conn"); }else{ - printf("success to connect server\n"); - lua_pushnumber(L, 0); + // printf("success to connect server\n"); + lua_pushinteger(L, 0); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); lua_setfield(L, table_index, "error"); lua_pushlightuserdata(L,taos); lua_setfield(L, table_index, "conn"); } - + return 1; } static int l_query(lua_State *L){ - TAOS * taos= lua_topointer(L,1); - char *s = lua_tostring(L, 2); + TAOS *taos= (TAOS*)lua_topointer(L,1); + const char* s = lua_tostring(L, 2); TAOS_RES *result; lua_newtable(L); int table_index = lua_gettop(L); // printf("receive command:%s\r\n",s); - result = taos_query(taos,s); - int32_t code = taos_errno(result); + result = taos_query(taos, s); + int32_t code = taos_errno(result); if( code != 0){ printf("failed, reason:%s\n", taos_errstr(result)); - lua_pushnumber(L, -1); + lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); lua_setfield(L, table_index, "error"); @@ -74,12 +106,12 @@ static int l_query(lua_State *L){ TAOS_ROW row; int rows = 0; int num_fields = taos_field_count(result); - TAOS_FIELD *fields = taos_fetch_fields(result); - char temp[256]; + const TAOS_FIELD *fields = taos_fetch_fields(result); + //char temp[256]; - int affectRows = taos_affected_rows(result); + const int affectRows = taos_affected_rows(result); // printf(" affect rows:%d\r\n", affectRows); - lua_pushnumber(L, 0); + lua_pushinteger(L, 0); lua_setfield(L, table_index, "code"); lua_pushinteger(L, affectRows); lua_setfield(L, table_index, "affected"); @@ -150,8 +182,8 @@ void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ TAOS_FIELD *fields = taos_fetch_fields(result); int numFields = taos_num_fields(result); - printf("\nnumfields:%d\n", numFields); - printf("\n\r-----------------------------------------------------------------------------------\n"); + // printf("\nnumfields:%d\n", numFields); + //printf("\n\r-----------------------------------------------------------------------------------\n"); lua_State *L = p->state; lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); @@ -204,13 +236,13 @@ void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ lua_call(L, 1, 0); - printf("-----------------------------------------------------------------------------------\n\r"); + // printf("-----------------------------------------------------------------------------------\n\r"); } static int l_open_stream(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); - TAOS * taos = lua_topointer(L,1); - char * sqlstr = lua_tostring(L,2); + TAOS * taos = (TAOS*)lua_topointer(L,1); + const char * sqlstr = lua_tostring(L,2); int stime = luaL_checknumber(L,3); lua_newtable(L); @@ -253,7 +285,7 @@ static int l_close_stream(lua_State *L){ } static int l_close(lua_State *L){ - TAOS * taos= lua_topointer(L,1); + TAOS *taos= (TAOS*)lua_topointer(L,1); lua_newtable(L); int table_index = lua_gettop(L); @@ -263,7 +295,7 @@ static int l_close(lua_State *L){ lua_pushstring(L, "null pointer."); lua_setfield(L, table_index, "error"); }else{ - taos_close(taos); + taos_close(taos); lua_pushnumber(L, 0); lua_setfield(L, table_index, "code"); lua_pushstring(L, "done."); diff --git a/tests/examples/lua/test.lua b/tests/examples/lua/test.lua index 4d5f9fe7d36f99f9a9452e8137bf298361a05685..9f9c6934aa46d52e1578700b067193351120dbab 100644 --- a/tests/examples/lua/test.lua +++ b/tests/examples/lua/test.lua @@ -1,93 +1,117 @@ local driver = require "luaconnector" -local host="127.0.0.1" -local user="root" -local password="taosdata" -local db =nil -local port=6030 -local conn +local config = { + host = "127.0.0.1", + port = 6030, + database = "", + user = "root", + password = "taosdata", + max_packet_size = 1024 * 1024 +} -local res = driver.connect(host,user,password,db,port) +local conn +local res = driver.connect(config) if res.code ~=0 then - print(res.error) + print("connect--- failed: "..res.error) return else conn = res.conn + print("connect--- pass.") end local res = driver.query(conn,"drop database if exists demo") res = driver.query(conn,"create database demo") if res.code ~=0 then - print(res.error) + print("create db--- failed: "..res.error) return +else + print("create db--- pass.") end res = driver.query(conn,"use demo") if res.code ~=0 then - print(res.error) + print("select db--- failed: "..res.error) return +else + print("select db--- pass.") end res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") if res.code ~=0 then - print(res.error) + print("create table---failed: "..res.error) return +else + print("create table--- pass.") end res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") if res.code ~=0 then - print(res.error) + print("insert records failed: "..res.error) return else - print("insert successfully, affected:"..res.affected) + if(res.affected == 3) then + print("insert records--- pass") + else + print("insert records---failed: expect 3 affected records, actually affected "..res.affected) + end end res = driver.query(conn,"select * from m1") if res.code ~=0 then - print("select error:"..res.error) + print("select failed: "..res.error) return else - print("in lua, result:") - for i = 1, #(res.item) do - print("timestamp:"..res.item[i].ts) - print("speed:"..res.item[i].speed) - print("owner:"..res.item[i].owner) - end + if (#(res.item) == 3) then + print("select--- pass") + else + print("select--- failed: expect 3 affected records, actually received "..#(res.item)) + end + end res = driver.query(conn,"CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int)") if res.code ~=0 then print(res.error) return +else + print("create super table--- pass") end res = driver.query(conn,"CREATE TABLE therm1 USING thermometer TAGS ('beijing', 1)") if res.code ~=0 then print(res.error) return +else + print("create table--- pass") end + res = driver.query(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") if res.code ~=0 then print(res.error) return else - print("insert successfully, affected:"..res.affected) + if(res.affected == 2) then + print("insert records--- pass") + else + print("insert records---failed: expect 2 affected records, actually affected "..res.affected) + end end res = driver.query(conn,"SELECT COUNT(*) count, AVG(degree) AS av, MAX(degree), MIN(degree) FROM thermometer WHERE location='beijing' or location='tianjin' GROUP BY location, type") if res.code ~=0 then - print("select error:"..res.error) + print("select from super table--- failed:"..res.error) return else - print("in lua, result:") + print("select from super table--- pass") for i = 1, #(res.item) do print("res:"..res.item[i].count) end end function callback(t) + print("------------------------") print("continuous query result:") for key, value in pairs(t) do print("key:"..key..", value:"..value) @@ -97,25 +121,25 @@ end local stream res = driver.open_stream(conn,"SELECT COUNT(*) as count, AVG(degree) as avg, MAX(degree) as max, MIN(degree) as min FROM thermometer interval(2s) sliding(2s);)",0,callback) if res.code ~=0 then - print("open stream error:"..res.error) + print("open stream--- failed:"..res.error) return else - print("openstream ok") + print("open stream--- pass") stream = res.stream end ---From now on we begin continous query in an definite (infinite if you want) loop. +print("From now on we start continous insert in an definite (infinite if you want) loop.") local loop_index = 0 -while loop_index < 10 do +while loop_index < 30 do local t = os.time()*1000 local v = loop_index res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) if res.code ~=0 then - print(res.error) + print("continous insertion--- failed:" .. res.error) return else - print("insert successfully, affected:"..res.affected) + --print("insert successfully, affected:"..res.affected) end os.execute("sleep " .. 1) loop_index = loop_index + 1 diff --git a/tests/examples/nodejs/nodejsChecker.js b/tests/examples/nodejs/nodejsChecker.js index f838d5cc8465dba70b5372a5d7720a8cff69544a..e634a54ea1c56310bae7d38188590a0d7862f953 100644 --- a/tests/examples/nodejs/nodejsChecker.js +++ b/tests/examples/nodejs/nodejsChecker.js @@ -1,4 +1,5 @@ const taos = require('td2.0-connector'); +//const taos = require('../../../src/connector/nodejs/'); var host = null; diff --git a/tests/examples/python/taosdemo/README.md b/tests/examples/python/taosdemo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d48fffe8ff44fb68a1a147a2c97ca057fb360092 --- /dev/null +++ b/tests/examples/python/taosdemo/README.md @@ -0,0 +1,38 @@ +install build environment +=== +/usr/bin/python3 -m pip install -r requirements.txt + +run python version taosdemo +=== +Usage: ./taosdemo.py [OPTION...] + +Author: Shuduo Sang + + -H, --help Show usage. + + -N, --native flag, Use native interface if set. Default is using RESTful interface. + -h, --host host, The host to connect to TDengine. Default is localhost. + -p, --port port, The TCP/IP port number to use for the connection. Default is 0. + -u, --user user, The user name to use when connecting to the server. Default is 'root'. + -P, --password password, The password to use when connecting to the server. Default is 'taosdata'. + -l, --colsPerRec num_of_columns_per_record, The number of columns per record. Default is 3. + -d, --dbname database, Destination database. Default is 'test'. + -a, --replica replica, Set the replica parameters of the database, Default 1, min: 1, max: 5. + -m, --tbname
    table_prefix, Table prefix name. Default is 't'. + -M, --stable flag, Use super table. Default is no + -s, --stbname stable_prefix, STable prefix name. Default is 'st' + -Q, --query query, Execute query command. set 'DEFAULT' means select * from each table + -T, --threads num_of_threads, The number of threads. Default is 1. + -C, --processes num_of_processes, The number of threads. Default is 1. + -r, --batch num_of_records_per_req, The number of records per request. Default is 1000. + -t, --numOfTb num_of_tables, The number of tables. Default is 1. + -n, --numOfRec num_of_records_per_table, The number of records per table. Default is 1. + -c, --config config_directory, Configuration directory. Default is '/etc/taos/'. + -x, --inserOnly flag, Insert only flag. + -O, --outOfOrder out of order data insert, 0: In order, 1: Out of order. Default is in order. + -R, --rateOOOO rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50. + -D, --deleteMethod Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database. + -v, --verbose Print verbose output + -g, --debug Print debug output + -y, --skipPrompt Skip read key for continous test, default is not skip + diff --git a/tests/examples/python/taosdemo/requirements.txt b/tests/examples/python/taosdemo/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..977e8e3726a446e85635764fe9243a3c5416ea0f --- /dev/null +++ b/tests/examples/python/taosdemo/requirements.txt @@ -0,0 +1,28 @@ +## +######## example-requirements.txt ####### +## +####### Requirements without Version Specifiers ###### +requests +multipledispatch +#beautifulsoup4 +## +####### Requirements with Version Specifiers ###### +## See https://www.python.org/dev/peps/pep-0440/#version-specifiers +#docopt == 0.6.1 # Version Matching. Must be version 0.6.1 +#keyring >= 4.1.1 # Minimum version 4.1.1 +#coverage != 3.5 # Version Exclusion. Anything except version 3.5 +#Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.* +## +####### Refer to other requirements files ###### +#-r other-requirements.txt +## +## +####### A particular file ###### +#./downloads/numpy-1.9.2-cp34-none-win32.whl +#http://wxpython.org/Phoenix/snapshot-builds/wxPython_Phoenix-3.0.3.dev1820+49a8884-cp34-none-win_amd64.whl +## +####### Additional Requirements without Version Specifiers ###### +## Same as 1st section, just here to show that you can put things in any order. +#rejected +#green +## diff --git a/tests/examples/python/taosdemo/taosdemo.py b/tests/examples/python/taosdemo/taosdemo.py new file mode 100755 index 0000000000000000000000000000000000000000..d55023bdbf119544a788aa6246c9d63dbf024872 --- /dev/null +++ b/tests/examples/python/taosdemo/taosdemo.py @@ -0,0 +1,797 @@ +#!/usr/bin/python3 +# * Copyright (c) 2019 TAOS Data, Inc. +# * +# * This program is free software: you can use, redistribute, and/or modify +# * it under the terms of the GNU Affero General Public License, version 3 +# * or later ("AGPL"), as published by the Free Software Foundation. +# * +# * This program is distributed in the hope that it will be useful, but WITHOUT +# * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# * FITNESS FOR A PARTICULAR PURPOSE. +# * +# * You should have received a copy of the GNU Affero General Public License +# * along with this program. If not, see . + +# -*- coding: utf-8 -*- + +import sys +import getopt +import requests +import json +import random +import time +import datetime +from multiprocessing import Manager, Pool, Lock +from multipledispatch import dispatch +from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED + + +@dispatch(str, str) +def v_print(msg: str, arg: str): + if verbose: + print(msg % arg) + + +@dispatch(str, str, str) +def v_print(msg: str, arg1: str, arg2: str): + if verbose: + print(msg % (arg1, arg2)) + + +@dispatch(str, str, str, str) +def v_print(msg: str, arg1: str, arg2: str, arg3: str): + if verbose: + print(msg % (arg1, arg2, arg3)) + + +@dispatch(str, str, str, str, str) +def v_print(msg: str, arg1: str, arg2: str, arg3: str, arg4: str): + if verbose: + print(msg % (arg1, arg2, arg3, arg4)) + + +@dispatch(str, int) +def v_print(msg: str, arg: int): + if verbose: + print(msg % int(arg)) + + +@dispatch(str, int, str) +def v_print(msg: str, arg1: int, arg2: str): + if verbose: + print(msg % (int(arg1), str(arg2))) + + +@dispatch(str, str, int) +def v_print(msg: str, arg1: str, arg2: int): + if verbose: + print(msg % (arg1, int(arg2))) + + +@dispatch(str, int, int) +def v_print(msg: str, arg1: int, arg2: int): + if verbose: + print(msg % (int(arg1), int(arg2))) + + +@dispatch(str, int, int, str) +def v_print(msg: str, arg1: int, arg2: int, arg3: str): + if verbose: + print(msg % (int(arg1), int(arg2), str(arg3))) + + +@dispatch(str, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3))) + + +@dispatch(str, int, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) + + +def restful_execute(host: str, port: int, user: str, password: str, cmd: str): + url = "http://%s:%d/rest/sql" % (host, restPort) + + v_print("restful_execute - cmd: %s", cmd) + + resp = requests.post(url, cmd, auth=(user, password)) + + v_print("resp status: %d", resp.status_code) + + if debug: + v_print( + "resp text: %s", + json.dumps( + resp.json(), + sort_keys=True, + indent=2)) + else: + print("resp: %s" % json.dumps(resp.json())) + + +def query_func(process: int, thread: int, cmd: str): + v_print("%d process %d thread cmd: %s", process, thread, cmd) + + if oneMoreHost != "NotSupported" and random.randint( + 0, 1) == 1: + v_print("%s", "Send to second host") + if native: + cursor2.execute(cmd) + else: + restful_execute( + oneMoreHost, port, user, password, cmd) + else: + v_print("%s%s%s", "Send ", cmd, " to the host") + if native: + pass +# cursor.execute(cmd) + else: + restful_execute( + host, port, user, password, cmd) + + +def query_data_process(cmd: str): + # establish connection if native + if native: + v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + try: + conn = taos.connect( + host=host, + user=user, + password=password, + config=configDir) + v_print("conn: %s", str(conn.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + + try: + cursor = conn.cursor() + v_print("cursor:%d %s", id(cursor), str(cursor.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + conn.close() + sys.exit(1) + + if native: + try: + cursor.execute(cmd) + cols = cursor.description + data = cursor.fetchall() + + for col in data: + print(col) + except Exception as e: + conn.close() + print("Error: %s" % e.args[0]) + sys.exit(1) + + else: + restful_execute( + host, + port, + user, + password, + cmd) + + if native: + cursor.close() + conn.close() + + +def create_stb(): + for i in range(0, numOfStb): + if native: + cursor.execute( + "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" % + (stbName, i)) + else: + restful_execute( + host, + port, + user, + password, + "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" % + (stbName, i) + ) + + +def use_database(): + + if native: + cursor.execute("USE %s" % current_db) + else: + restful_execute(host, port, user, password, "USE %s" % current_db) + + +def create_databases(): + for i in range(0, numOfDb): + v_print("will create database db%d", int(i)) + + if native: + cursor.execute( + "CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i)) + else: + restful_execute( + host, + port, + user, + password, + "CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i)) + + +def drop_tables(): + # TODO + v_print("TODO: drop tables total %d", numOfTb) + pass + + +def drop_stable(): + # TODO + v_print("TODO: drop stables total %d", numOfStb) + pass + + +def drop_databases(): + v_print("drop databases total %d", numOfDb) + + # drop exist databases first + for i in range(0, numOfDb): + v_print("will drop database db%d", int(i)) + + if native: + cursor.execute( + "DROP DATABASE IF EXISTS %s%d" % + (dbName, i)) + else: + restful_execute( + host, + port, + user, + password, + "DROP DATABASE IF EXISTS %s%d" % + (dbName, i)) + + +def insert_func(process: int, thread: int): + v_print("%d process %d thread, insert_func ", process, thread) + + # generate uuid + uuid_int = random.randint(0, numOfTb + 1) + uuid = "%s" % uuid_int + v_print("uuid is: %s", uuid) + + # establish connection if native + if native: + v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + try: + conn = taos.connect( + host=host, + user=user, + password=password, + config=configDir) + v_print("conn: %s", str(conn.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + + try: + cursor = conn.cursor() + v_print("cursor:%d %s", id(cursor), str(cursor.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + conn.close() + sys.exit(1) + + v_print("numOfRec %d:", numOfRec) + + row = 0 + while row < numOfRec: + v_print("row: %d", row) + sqlCmd = ['INSERT INTO '] + try: + sqlCmd.append( + "%s.%s%d " % (current_db, tbName, thread)) + + if (numOfStb > 0 and autosubtable): + sqlCmd.append("USING %s.%s%d TAGS('%s') " % + (current_db, stbName, numOfStb - 1, uuid)) + + start_time = datetime.datetime( + 2021, 1, 25) + datetime.timedelta(seconds=row) + + sqlCmd.append("VALUES ") + for batchIter in range(0, batch): + sqlCmd.append("('%s', %f) " % + ( + start_time + + datetime.timedelta( + milliseconds=batchIter), + random.random())) + row = row + 1 + if row >= numOfRec: + v_print("BREAK, row: %d numOfRec:%d", row, numOfRec) + break + + except Exception as e: + print("Error: %s" % e.args[0]) + + cmd = ' '.join(sqlCmd) + + if measure: + exec_start_time = datetime.datetime.now() + + if native: + affectedRows = cursor.execute(cmd) + else: + restful_execute( + host, port, user, password, cmd) + + if measure: + exec_end_time = datetime.datetime.now() + exec_delta = exec_end_time - exec_start_time + v_print( + "consume %d microseconds", + exec_delta.microseconds) + + v_print("cmd: %s, length:%d", cmd, len(cmd)) + + if native: + cursor.close() + conn.close() + + +def create_tb_using_stb(): + # TODO: + pass + + +def create_tb(): + v_print("create_tb() numOfTb: %d", numOfTb) + for i in range(0, numOfDb): + if native: + cursor.execute("USE %s%d" % (dbName, i)) + else: + restful_execute( + host, port, user, password, "USE %s%d" % + (dbName, i)) + + for j in range(0, numOfTb): + if native: + cursor.execute( + "CREATE TABLE %s%d (ts timestamp, value float)" % + (tbName, j)) + else: + restful_execute( + host, + port, + user, + password, + "CREATE TABLE %s%d (ts timestamp, value float)" % + (tbName, j)) + + +def insert_data_process(lock, i: int, begin: int, end: int): + lock.acquire() + tasks = end - begin + v_print("insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks) + + if (threads < (end - begin)): + for j in range(begin, end, threads): + with ThreadPoolExecutor(max_workers=threads) as executor: + k = end if ((j + threads) > end) else (j + threads) + workers = [ + executor.submit( + insert_func, + i, + n) for n in range( + j, + k)] + wait(workers, return_when=ALL_COMPLETED) + else: + with ThreadPoolExecutor(max_workers=threads) as executor: + workers = [ + executor.submit( + insert_func, + i, + j) for j in range( + begin, + end)] + wait(workers, return_when=ALL_COMPLETED) + + lock.release() + + +def query_db(i): + if native: + cursor.execute("USE %s%d" % (dbName, i)) + else: + restful_execute( + host, port, user, password, "USE %s%d" % + (dbName, i)) + + for j in range(0, numOfTb): + if native: + cursor.execute( + "SELECT COUNT(*) FROM %s%d" % (tbName, j)) + else: + restful_execute( + host, port, user, password, "SELECT COUNT(*) FROM %s%d" % + (tbName, j)) + + +def printConfig(): + + print("###################################################################") + print("# Use native interface: %s" % native) + print("# Server IP: %s" % host) + if native: + print("# Server port: %s" % port) + else: + print("# Server port: %s" % restPort) + + print("# Configuration Dir: %s" % configDir) + print("# User: %s" % user) + print("# Password: %s" % password) + print("# Number of Columns per record: %s" % colsPerRecord) + print("# Number of Threads: %s" % threads) + print("# Number of Processes: %s" % processes) + print("# Number of Tables: %s" % numOfTb) + print("# Number of records per Table: %s" % numOfRec) + print("# Records/Request: %s" % batch) + print("# Database name: %s" % dbName) + print("# Replica: %s" % replica) + print("# Use STable: %s" % useStable) + print("# Table prefix: %s" % tbName) + if useStable: + print("# STable prefix: %s" % stbName) + + print("# Data order: %s" % outOfOrder) + print("# Data out of order rate: %s" % rateOOOO) + print("# Delete method: %s" % deleteMethod) + print("# Query command: %s" % queryCmd) + print("# Insert Only: %s" % insertOnly) + print("# Verbose output %s" % verbose) + print("# Test time: %s" % + datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + print("###################################################################") + + +if __name__ == "__main__": + + native = False + verbose = False + debug = False + measure = True + dropDbOnly = False + colsPerRecord = 3 + numOfDb = 1 + dbName = "test" + replica = 1 + batch = 1 + numOfTb = 1 + tbName = "tb" + useStable = False + numOfStb = 0 + stbName = "stb" + numOfRec = 10 + ieration = 1 + host = "127.0.0.1" + configDir = "/etc/taos" + oneMoreHost = "NotSupported" + port = 6030 + restPort = 6041 + user = "root" + defaultPass = "taosdata" + processes = 1 + threads = 1 + insertOnly = False + autosubtable = False + queryCmd = "NO" + outOfOrder = 0 + rateOOOO = 0 + deleteMethod = 0 + skipPrompt = False + + try: + opts, args = getopt.gnu_getopt(sys.argv[1:], + 'Nh:p:u:P:d:a:m:Ms:Q:T:C:r:l:t:n:c:xOR:D:vgyH', + [ + 'native', 'host', 'port', 'user', 'password', 'dbname', 'replica', 'tbname', + 'stable', 'stbname', 'query', 'threads', 'processes', + 'recPerReq', 'colsPerRecord', 'numOfTb', 'numOfRec', 'config', + 'insertOnly', 'outOfOrder', 'rateOOOO', 'deleteMethod', + 'verbose', 'debug', 'skipPrompt', 'help' + ]) + except getopt.GetoptError as err: + print('ERROR:', err) + print('Try `taosdemo.py --help` for more options.') + sys.exit(1) + + if bool(opts) is False: + print('Try `taosdemo.py --help` for more options.') + sys.exit(1) + + for key, value in opts: + if key in ['-H', '--help']: + print('') + print( + 'taosdemo.py for TDengine') + print('') + print('Author: Shuduo Sang ') + print('') + + print('\t-H, --help Show usage.') + print('') + + print('\t-N, --native flag, Use native interface if set. Default is using RESTful interface.') + print('\t-h, --host host, The host to connect to TDengine. Default is localhost.') + print('\t-p, --port port, The TCP/IP port number to use for the connection. Default is 0.') + print('\t-u, --user user, The user name to use when connecting to the server. Default is \'root\'.') + print('\t-P, --password password, The password to use when connecting to the server. Default is \'taosdata\'.') + print('\t-l, --colsPerRec num_of_columns_per_record, The number of columns per record. Default is 3.') + print( + '\t-d, --dbname database, Destination database. Default is \'test\'.') + print('\t-a, --replica replica, Set the replica parameters of the database, Default 1, min: 1, max: 5.') + print( + '\t-m, --tbname
    table_prefix, Table prefix name. Default is \'t\'.') + print( + '\t-M, --stable flag, Use super table. Default is no') + print( + '\t-s, --stbname stable_prefix, STable prefix name. Default is \'st\'') + print('\t-Q, --query [NO|EACHTB|command] query, Execute query command. set \'EACHTB\' means select * from each table') + print( + '\t-T, --threads num_of_threads, The number of threads. Default is 1.') + print( + '\t-C, --processes num_of_processes, The number of threads. Default is 1.') + print('\t-r, --batch num_of_records_per_req, The number of records per request. Default is 1000.') + print( + '\t-t, --numOfTb num_of_tables, The number of tables. Default is 1.') + print('\t-n, --numOfRec num_of_records_per_table, The number of records per table. Default is 1.') + print('\t-c, --config config_directory, Configuration directory. Default is \'/etc/taos/\'.') + print('\t-x, --inserOnly flag, Insert only flag.') + print('\t-O, --outOfOrder out of order data insert, 0: In order, 1: Out of order. Default is in order.') + print('\t-R, --rateOOOO rate, Out of order data\'s rate--if order=1 Default 10, min: 0, max: 50.') + print('\t-D, --deleteMethod Delete data methods 0: don\'t delete, 1: delete by table, 2: delete by stable, 3: delete by database.') + print('\t-v, --verbose Print verbose output') + print('\t-g, --debug Print debug output') + print( + '\t-y, --skipPrompt Skip read key for continous test, default is not skip') + print('') + sys.exit(0) + + if key in ['-N', '--native']: + try: + import taos + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + native = True + + if key in ['-h', '--host']: + host = value + + if key in ['-p', '--port']: + port = int(value) + + if key in ['-u', '--user']: + user = value + + if key in ['-P', '--password']: + password = value + else: + password = defaultPass + + if key in ['-d', '--dbname']: + dbName = value + + if key in ['-a', '--replica']: + replica = int(value) + if replica < 1: + print("FATAL: number of replica need > 0") + sys.exit(1) + + if key in ['-m', '--tbname']: + tbName = value + + if key in ['-M', '--stable']: + useStable = True + numOfStb = 1 + + if key in ['-s', '--stbname']: + stbName = value + + if key in ['-Q', '--query']: + queryCmd = str(value) + + if key in ['-T', '--threads']: + threads = int(value) + if threads < 1: + print("FATAL: number of threads must be larger than 0") + sys.exit(1) + + if key in ['-C', '--processes']: + processes = int(value) + if processes < 1: + print("FATAL: number of processes must be larger than 0") + sys.exit(1) + + if key in ['-r', '--batch']: + batch = int(value) + + if key in ['-l', '--colsPerRec']: + colsPerRec = int(value) + + if key in ['-t', '--numOfTb']: + numOfTb = int(value) + v_print("numOfTb is %d", numOfTb) + + if key in ['-n', '--numOfRec']: + numOfRec = int(value) + v_print("numOfRec is %d", numOfRec) + if numOfRec < 1: + print("FATAL: number of records must be larger than 0") + sys.exit(1) + + + if key in ['-c', '--config']: + configDir = value + v_print("config dir: %s", configDir) + + if key in ['-x', '--insertOnly']: + insertOnly = True + v_print("insert only: %d", insertOnly) + + if key in ['-O', '--outOfOrder']: + outOfOrder = int(value) + v_print("out of order is %d", outOfOrder) + + if key in ['-R', '--rateOOOO']: + rateOOOO = int(value) + v_print("the rate of out of order is %d", rateOOOO) + + if key in ['-D', '--deleteMethod']: + deleteMethod = int(value) + if (deleteMethod < 0) or (deleteMethod > 3): + print( + "inputed delete method is %d, valid value is 0~3, set to default 0" % + deleteMethod) + deleteMethod = 0 + v_print("the delete method is %d", deleteMethod) + + if key in ['-v', '--verbose']: + verbose = True + + if key in ['-g', '--debug']: + debug = True + + if key in ['-y', '--skipPrompt']: + skipPrompt = True + + if verbose: + printConfig() + + if not skipPrompt: + input("Press any key to continue..") + + # establish connection first if native + if native: + v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + try: + conn = taos.connect( + host=host, + user=user, + password=password, + config=configDir) + v_print("conn: %s", str(conn.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + + try: + cursor = conn.cursor() + v_print("cursor:%d %s", id(cursor), str(cursor.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + conn.close() + sys.exit(1) + + # drop data only if delete method be set + if deleteMethod > 0: + if deleteMethod == 1: + drop_tables() + print("Drop tables done.") + elif deleteMethod == 2: + drop_stables() + print("Drop super tables done.") + elif deleteMethod == 3: + drop_databases() + print("Drop Database done.") + sys.exit(0) + + # create databases + drop_databases() + create_databases() + + # use last database + current_db = "%s%d" % (dbName, (numOfDb - 1)) + use_database() + + if measure: + start_time_begin = time.time() + + if numOfStb > 0: + create_stb() + if (autosubtable == False): + create_tb_using_stb() + else: + create_tb() + + if measure: + end_time = time.time() + print( + "Total time consumed {} seconds for create table.".format( + (end_time - start_time_begin))) + + if native: + cursor.close() + conn.close() + + # start insert data + if measure: + start_time = time.time() + + manager = Manager() + lock = manager.Lock() + pool = Pool(processes) + + begin = 0 + end = 0 + + quotient = numOfTb // processes + if quotient < 1: + processes = numOfTb + quotient = 1 + + remainder = numOfTb % processes + v_print( + "num of tables: %d, quotient: %d, remainder: %d", + numOfTb, + quotient, + remainder) + + for i in range(processes): + begin = end + + if i < remainder: + end = begin + quotient + 1 + else: + end = begin + quotient + pool.apply_async(insert_data_process, args=(lock, i, begin, end,)) + + pool.close() + pool.join() + time.sleep(1) + + if measure: + end_time = time.time() + print( + "Total time consumed {} seconds for insert data.".format( + (end_time - start_time))) + + + # query data + if queryCmd != "NO": + print("queryCmd: %s" % queryCmd) + query_data_process(queryCmd) + + if measure: + end_time = time.time() + print( + "Total time consumed {} seconds.".format( + (end_time - start_time_begin))) + + print("done") diff --git a/tests/examples/rust b/tests/examples/rust new file mode 160000 index 0000000000000000000000000000000000000000..1c8924dc668e6aa848214c2fc54e3ace3f5bf8df --- /dev/null +++ b/tests/examples/rust @@ -0,0 +1 @@ +Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df diff --git a/tests/perftest-scripts/coverage_test.sh b/tests/perftest-scripts/coverage_test.sh index 5085ec89d00def1a2147f24ccbeb488406541775..1be2053f24f28dd825c040912783675b1eab94f9 100755 --- a/tests/perftest-scripts/coverage_test.sh +++ b/tests/perftest-scripts/coverage_test.sh @@ -56,9 +56,9 @@ function runGeneralCaseOneByOne { case=`echo $line | grep sim$ |awk '{print $NF}'` if [ -n "$case" ]; then - ./test.sh -f $case > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \ - echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT ) \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT fi fi done < $1 diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 51bb9b36c3c55802fa904de223c2ae4ea2ea7151..e43a45ed5e87ec81fc86e1af4733c0e859af69f0 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -39,6 +39,8 @@ function buildTDengine { cd $WORK_DIR/TDengine git remote update > /dev/null + git reset --hard HEAD + git checkout develop REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` LOCAL_COMMIT=`git rev-parse --short @` @@ -54,18 +56,36 @@ function buildTDengine { cd debug rm -rf * cmake .. > /dev/null - make > /dev/null - make install + make && make install > /dev/null fi } function runQueryPerfTest { [ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 & - echoInfo "Run Performance Test" + echoInfo "Wait TDengine to start" + sleep 120 + echoInfo "Run Performance Test" cd $WORK_DIR/TDengine/tests/pytest - python3 query/queryPerformance.py 0 | tee -a $PERFORMANCE_TEST_REPORT + python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT + + python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT + + yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -x > taosdemoperf.txt + + CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` + INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` + REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` + AVGDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` + MAXDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'` + MINDELAY=`echo ${delay:0:${#delay}-2}` + + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT + [ -f taosdemoperf.txt ] && rm taosdemoperf.txt } @@ -87,6 +107,7 @@ function sendReport { stopTaosd buildTDengine runQueryPerfTest +stopTaosd echoInfo "Send Report" sendReport diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/tests/perftest-scripts/perftest-taosdemo.sh index ae9e8dd7371cca2f522810da6d7816adba86a862..dffa251ef76d866f33247bd0ce6fd5bb718449de 100755 --- a/tests/perftest-scripts/perftest-taosdemo.sh +++ b/tests/perftest-scripts/perftest-taosdemo.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes function red_echo { echo -e "\033[31m$@\033[0m"; } function green_echo { echo -e "\033[32m$@\033[0m"; } diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh index 110ab9e5fa6463db9f55c8320b1fd3e36380d7a1..3f56a048e9f78b9e6d02bf8f1d704d27d46e1149 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh index 9e8bc697bc18e0e5c3a6d5957fed948192c09241..29e72d7b3ff62aa30fdefd694e435335e752e630 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh index 5fc5b9d03a4339263eb07a9af9b76ee2fc324f1b..3bc63b831b901b9a8a4b9b96385574a345ab58c6 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh index bafa04f174955d9938810f9b5bc353a0b712c3c3..566479c967320e9bbb8a9e877d26e1bbeb750add 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/pytest/account/account_create.py b/tests/pytest/account/account_create.py index 85adfff19994a3fbb12611e28d9872fb6cb1aeca..c008acccd8d8d33af0ca77bd161d4ab020c1543e 100644 --- a/tests/pytest/account/account_create.py +++ b/tests/pytest/account/account_create.py @@ -24,6 +24,17 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): + tdSql.query("show users") + rows = tdSql.queryRows + + tdSql.execute("create user test PASS 'test' ") + tdSql.query("show users") + tdSql.checkRows(rows + 1) + + tdSql.error("create user tdenginetdenginetdengine PASS 'test' ") + + tdSql.error("create user tdenginet PASS '1234512345123456' ") + try: tdSql.execute("create account a&cc PASS 'pass123'") except Exception as e: @@ -31,6 +42,7 @@ class TDTestCase: return tdLog.exit("drop built-in user is error.") + def stop(self): tdSql.close() diff --git a/tests/pytest/alter/alter_debugFlag.py b/tests/pytest/alter/alter_debugFlag.py new file mode 100644 index 0000000000000000000000000000000000000000..38d972b58252a704420af0dda7c091c22c894ecd --- /dev/null +++ b/tests/pytest/alter/alter_debugFlag.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + flagList=["debugflag", "cdebugflag", "tmrDebugFlag", "uDebugFlag", "rpcDebugFlag"] + + for flag in flagList: + tdSql.execute("alter local %s 131" % flag) + tdSql.execute("alter local %s 135" % flag) + tdSql.execute("alter local %s 143" % flag) + randomFlag = random.randint(100, 250) + if randomFlag != 131 and randomFlag != 135 and randomFlag != 143: + tdSql.error("alter local %s %d" % (flag, randomFlag)) + + tdSql.query("show dnodes") + dnodeId = tdSql.getData(0, 0) + + for flag in flagList: + tdSql.execute("alter dnode %d %s 131" % (dnodeId, flag)) + tdSql.execute("alter dnode %d %s 135" % (dnodeId, flag)) + tdSql.execute("alter dnode %d %s 143" % (dnodeId, flag)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/alter/alter_stable.py b/tests/pytest/alter/alter_stable.py index 6852f4ef4ab6730f6423ba388d4f088ca229312e..6dad43313248388ab34a51f6dd40414f67ebdb7f 100644 --- a/tests/pytest/alter/alter_stable.py +++ b/tests/pytest/alter/alter_stable.py @@ -19,6 +19,10 @@ class TDTestCase: "double", "smallint", "tinyint", + "int unsigned", + "bigint unsigned", + "smallint unsigned", + "tinyint unsigned", "binary(10)", "nchar(10)", "timestamp"] diff --git a/tests/pytest/alter/alter_table.py b/tests/pytest/alter/alter_table.py index 6d4f72556b4d1d4ae1a439ab6a56b2925e3d63aa..a5acb7a73e0844ac2235590cbc8b7793ef40746c 100644 --- a/tests/pytest/alter/alter_table.py +++ b/tests/pytest/alter/alter_table.py @@ -19,6 +19,10 @@ class TDTestCase: "double", "smallint", "tinyint", + "int unsigned", + "bigint unsigned", + "smallint unsigned", + "tinyint unsigned", "binary(10)", "nchar(10)", "timestamp"] @@ -105,11 +109,11 @@ class TDTestCase: # Create db tdSql.execute("drop database if exists %s" % (db)) tdSql.execute("reset query cache") - tdSql.execute("create database %s maxrows 200 maxtables 4" % (db)) + tdSql.execute("create database %s maxrows 200" % (db)) tdSql.execute("use %s" % (db)) # Create a table with one colunm of int type and insert 300 rows - tdLog.info("Create table tb") + tdLog.info("create table tb") tdSql.execute("create table tb (ts timestamp, c1 int)") tdLog.info("Insert %d rows into tb" % (self.rowNum)) for k in range(1, self.rowNum + 1): @@ -119,20 +123,23 @@ class TDTestCase: # Alter tb and add a column of smallint type, then query tb to see if # all added column are NULL self.addColumnAndCount() - tdDnodes.stop(1) - time.sleep(5) - tdDnodes.start(1) - time.sleep(5) + tdDnodes.stop(1) + tdDnodes.start(1) tdSql.query(self.sqlHead + self.sqlTail) - for i in range(2, len(self.types) + 2): - tdSql.checkData(0, i, self.rowNum * (len(self.types) + 2 - i)) + size = len(self.types) + 2 + for i in range(2, size): + tdSql.checkData(0, i, self.rowNum * (size - i)) - self.dropColumnAndCount() + tdSql.execute("create table st(ts timestamp, c1 int) tags(t1 float)") + tdSql.execute("create table t0 using st tags(null)") + tdSql.execute("alter table t0 set tag t1=2.1") + + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) - -#tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/alter/alter_table_crash.py b/tests/pytest/alter/alter_table_crash.py index d1af022e35e636b15bc6e0fa175c7c41ed1ff640..2ec24860db9c078445cd186807a4c1834469f909 100644 --- a/tests/pytest/alter/alter_table_crash.py +++ b/tests/pytest/alter/alter_table_crash.py @@ -66,6 +66,14 @@ class TDTestCase: "alter table dt add column tbcol8 nchar(20)") tdSql.execute( "alter table dt add column tbcol9 binary(20)") + tdSql.execute( + "alter table dt add column tbcol10 tinyint unsigned") + tdSql.execute( + "alter table dt add column tbcol11 int unsigned") + tdSql.execute( + "alter table dt add column tbcol12 smallint unsigned") + tdSql.execute( + "alter table dt add column tbcol13 bigint unsigned") # restart taosd tdDnodes.forcestop(1) diff --git a/tests/pytest/bug2265.py b/tests/pytest/bug2265.py new file mode 100644 index 0000000000000000000000000000000000000000..e78233928fcd7f2348dc83c64b85cca3cf983477 --- /dev/null +++ b/tests/pytest/bug2265.py @@ -0,0 +1,85 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import taos +if __name__ == "__main__": + + logSql = True + deployPath = "" + testCluster = False + valgrind = 0 + + print("start to execute %s" % __file__) + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + + tdDnodes.stopAll() + tdDnodes.addSimExtraCfg("maxSQLLength", "1048576") + tdDnodes.deploy(1) + tdDnodes.start(1) + host = '127.0.0.1' + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + tdCases.logSql(logSql) + print('1') + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + + tdSql.init(conn.cursor(), True) + + print("==========step1") + print("create table ") + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute("create table t1 (ts timestamp, c1 int,c2 int ,c3 int)") + + print("==========step2") + print("insert maxSQLLength data ") + data = 'insert into t1 values' + ts = 1604298064000 + i = 0 + while ((len(data)<(1024*1024)) & (i < 32767 - 1) ): + data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000) + i+=1 + tdSql.execute(data) + + print("==========step4") + print("insert data batch larger than 32767 ") + i = 0 + while ((len(data)<(1024*1024)) & (i < 32767) ): + data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000) + i+=1 + tdSql.error(data) + + print("==========step4") + print("insert data larger than maxSQLLength ") + tdSql.execute("create table t2 (ts timestamp, c1 binary(50))") + data = 'insert into t2 values' + i = 0 + while ((len(data)<(1024*1024)) & (i < 32767 - 1 ) ): + data += '(%s,%s)'%(ts+i,'a'*50) + i+=1 + tdSql.error(data) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + + diff --git a/tests/pytest/client/alterDatabase.py b/tests/pytest/client/alterDatabase.py index 8191312cc060f9c20632911c956065f777d47079..bc8c4fc17ee956ad21525adb0587f925bb35d2bf 100644 --- a/tests/pytest/client/alterDatabase.py +++ b/tests/pytest/client/alterDatabase.py @@ -35,10 +35,9 @@ class TDTestCase: tdSql.execute("alter database db keep 365,365,365") tdSql.query("show databases") tdSql.checkData(0, 7, "365,365,365") - - tdSql.execute("alter database db quorum 2") - tdSql.query("show databases") - tdSql.checkData(0, 5, 2) + + tdSql.error("alter database db quorum 2") + tdSql.execute("alter database db blocks 100") tdSql.query("show databases") diff --git a/tests/pytest/client/noConnectionErrorTest.py b/tests/pytest/client/noConnectionErrorTest.py new file mode 100644 index 0000000000000000000000000000000000000000..2c13016cf19506a6cb76bef1a294e5ec1a9c2b84 --- /dev/null +++ b/tests/pytest/client/noConnectionErrorTest.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdDnodes.stop(1) + sql = "use db" + + try: + tdSql.execute(sql) + except Exception as e: + expectError = 'Unable to establish connection' + if expectError in str(e): + pass + else: + caller = inspect.getframeinfo(inspect.stack()[1][1]) + tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/cluster/clusterEnvSetup/Dockerfile b/tests/pytest/cluster/clusterEnvSetup/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c9c4d79be981e45609e040bf5835e275fc446260 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:latest AS builder + +ARG PACKAGE=TDengine-server-1.6.5.10-Linux-x64.tar.gz +ARG EXTRACTDIR=TDengine-enterprise-server +ARG TARBITRATORPKG=TDengine-tarbitrator-1.6.5.10-Linux-x64.tar.gz +ARG EXTRACTDIR2=TDengine-enterprise-arbitrator +ARG CONTENT=taos.tar.gz + +WORKDIR /root + +COPY ${PACKAGE} . +COPY ${TARBITRATORPKG} . + +RUN tar -zxf ${PACKAGE} +RUN tar -zxf ${TARBITRATORPKG} +RUN mv ${EXTRACTDIR}/driver ./lib +RUN tar -zxf ${EXTRACTDIR}/${CONTENT} +RUN mv ${EXTRACTDIR2}/bin/* /root/bin + +FROM ubuntu:latest + +WORKDIR /root + +RUN apt-get update +RUN apt-get install -y vim tmux net-tools +RUN echo 'alias ll="ls -l --color=auto"' >> /root/.bashrc +RUN ulimit -c unlimited + +COPY --from=builder /root/bin/taosd /usr/bin +COPY --from=builder /root/bin/tarbitrator /usr/bin +COPY --from=builder /root/bin/taos /usr/bin +COPY --from=builder /root/cfg/taos.cfg /etc/taos/ +COPY --from=builder /root/lib/libtaos.so.* /usr/lib/libtaos.so.1 + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LC_CTYPE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +EXPOSE 6030-6042/tcp 6060/tcp 6030-6039/udp + +# VOLUME [ "/var/lib/taos", "/var/log/taos", "/etc/taos" ] + +CMD [ "bash" ] \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/basic.py b/tests/pytest/cluster/clusterEnvSetup/basic.py new file mode 100644 index 0000000000000000000000000000000000000000..d9b8e9ce4a7bc144839334332268ac0f09f78f0d --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/basic.py @@ -0,0 +1,100 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import taos +import random +import argparse + +class BuildDockerCluser: + + def __init__(self, hostName, user, password, configDir, numOfNodes, clusterVersion): + self.hostName = hostName + self.user = user + self.password = password + self.configDir = configDir + self.numOfNodes = numOfNodes + self.clusterVersion = clusterVersion + + def getConnection(self): + self.conn = taos.connect( + host = self.hostName, + user = self.user, + password = self.password, + config = self.configDir) + + def createDondes(self): + self.cursor = self.conn.cursor() + for i in range(2, self.numOfNodes + 1): + self.cursor.execute("create dnode tdnode%d" % i) + + def startArbitrator(self): + print("start arbitrator") + os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator") + + def run(self): + if self.numOfNodes < 2 or self.numOfNodes > 5: + print("the number of nodes must be between 2 and 5") + exit(0) + os.system("./buildClusterEnv.sh -n %d -v %s" % (self.numOfNodes, self.clusterVersion)) + self.getConnection() + self.createDondes() + self.startArbitrator() + +parser = argparse.ArgumentParser() +parser.add_argument( + '-H', + '--host', + action='store', + default='tdnode1', + type=str, + help='host name to be connected (default: tdnode1)') +parser.add_argument( + '-u', + '--user', + action='store', + default='root', + type=str, + help='user (default: root)') +parser.add_argument( + '-p', + '--password', + action='store', + default='taosdata', + type=str, + help='password (default: taosdata)') +parser.add_argument( + '-c', + '--config-dir', + action='store', + default='/etc/taos', + type=str, + help='configuration directory (default: /etc/taos)') +parser.add_argument( + '-n', + '--num-of-nodes', + action='store', + default=2, + type=int, + help='number of nodes in the cluster (default: 2, min: 2, max: 5)') +parser.add_argument( + '-v', + '--version', + action='store', + default='2.0.14.1', + type=str, + help='the version of the cluster to be build, Default is 2.0.14.1') + +args = parser.parse_args() +cluster = BuildDockerCluser(args.host, args.user, args.password, args.config_dir, args.num_of_nodes, args.version) +cluster.run() \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh new file mode 100755 index 0000000000000000000000000000000000000000..968cdd1c1c81b9f6dba68bc2cca542038ada8606 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -0,0 +1,123 @@ +#!/bin/bash +echo "Executing buildClusterEnv.sh" +DOCKER_DIR=/data +CURR_DIR=`pwd` + +if [ $# != 4 ]; then + echo "argument list need input : " + echo " -n numOfNodes" + echo " -v version" + exit 1 +fi + +NUM_OF_NODES= +VERSION= +while getopts "n:v:" arg +do + case $arg in + n) + NUM_OF_NODES=$OPTARG + ;; + v) + VERSION=$OPTARG + ;; + ?) + echo "unkonwn argument" + ;; + esac +done + +function addTaoscfg { + for i in {1..5} + do + touch /data/node$i/cfg/taos.cfg + echo 'firstEp tdnode1:6030' > /data/node$i/cfg/taos.cfg + echo 'fqdn tdnode'$i >> /data/node$i/cfg/taos.cfg + echo 'arbitrator tdnode1:6042' >> /data/node$i/cfg/taos.cfg + done +} + +function createDIR { + for i in {1..5} + do + mkdir -p /data/node$i/data + mkdir -p /data/node$i/log + mkdir -p /data/node$i/cfg + mkdir -p /data/node$i/core + done +} + +function cleanEnv { + for i in {1..5} + do + echo /data/node$i/data/* + rm -rf /data/node$i/data/* + echo /data/node$i/log/* + rm -rf /data/node$i/log/* + done +} + +function prepareBuild { + + if [ -d $CURR_DIR/../../../../release ]; then + echo release exists + rm -rf $CURR_DIR/../../../../release/* + fi + + if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + cd $CURR_DIR/../../../../packaging + echo "generating TDeninger packages" + ./release.sh -v edge -n $VERSION >> /dev/null + + if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then + echo "no TDengine install package found" + exit 1 + fi + + if [ ! -e $CURR_DIR/../../../../release/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + echo "no arbitrator install package found" + exit 1 + fi + + cd $CURR_DIR/../../../../release + mv TDengine-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + mv TDengine-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + fi + + rm -rf $DOCKER_DIR/*.yml + cd $CURR_DIR + + cp *.yml $DOCKER_DIR + cp Dockerfile $DOCKER_DIR +} + +function clusterUp { + echo "docker compose start" + + cd $DOCKER_DIR + + if [ $NUM_OF_NODES -eq 2 ]; then + echo "create 2 dnodes" + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose up -d + fi + + if [ $NUM_OF_NODES -eq 3 ]; then + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml up -d + fi + + if [ $NUM_OF_NODES -eq 4 ]; then + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d + fi + + if [ $NUM_OF_NODES -eq 5 ]; then + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d + fi + + echo "docker compose finish" +} + +createDIR +cleanEnv +addTaoscfg +prepareBuild +clusterUp \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..cb35abd9a1497c92dee10e1e6fb95027fb21710c --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml @@ -0,0 +1,117 @@ +version: '3.7' + +services: + td2.0-node1: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} + - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} + image: 'tdengine:${VERSION}' + container_name: 'tdnode1' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + volumes: + # bind data directory + - type: bind + source: /data/node1/data + target: /var/lib/taos + # bind log directory + - type: bind + source: /data/node1/log + target: /var/log/taos + # bind configuration + - type: bind + source: /data/node1/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: /data/node1/core + target: /coredump + - type: bind + source: /data + target: /root + hostname: tdnode1 + networks: + taos_update_net: + ipv4_address: 172.27.0.7 + command: taosd + + td2.0-node2: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - EXTRACTDIR=${DIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode2' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode1:172.27.0.7" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + volumes: + # bind data directory + - type: bind + source: /data/node2/data + target: /var/lib/taos + # bind log directory + - type: bind + source: /data/node2/log + target: /var/log/taos + # bind configuration + - type: bind + source: /data/node2/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: /data/node2/core + target: /coredump + - type: bind + source: /data + target: /root + hostname: tdnode2 + networks: + taos_update_net: + ipv4_address: 172.27.0.8 + command: taosd + + +networks: + taos_update_net: +# external: true + ipam: + driver: default + config: + - subnet: "172.27.0.0/24" diff --git a/tests/pytest/cluster/clusterEnvSetup/insert.json b/tests/pytest/cluster/clusterEnvSetup/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..56a64b7b8561877cb26b4ef2336ab8b98f26c02c --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/insert.json @@ -0,0 +1,55 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 1, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 2, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 1, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/cluster/clusterEnvSetup/node3.yml b/tests/pytest/cluster/clusterEnvSetup/node3.yml new file mode 100644 index 0000000000000000000000000000000000000000..4f4f3a6f991f0ffff51265fee4c5a3b8941b5d85 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/node3.yml @@ -0,0 +1,54 @@ +version: '3.7' + +services: + td2.0-node3: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - EXTRACTDIR=${DIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode3' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode1:172.27.0.7" + - "tdnode2:172.27.0.8" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + volumes: + # bind data directory + - type: bind + source: /data/node3/data + target: /var/lib/taos + # bind log directory + - type: bind + source: /data/node3/log + target: /var/log/taos + # bind configuration + - type: bind + source: /data/node3/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: /data/node3/core + target: /coredump + - type: bind + source: /data + target: /root + hostname: tdnode3 + networks: + taos_update_net: + ipv4_address: 172.27.0.9 + command: taosd \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/node4.yml b/tests/pytest/cluster/clusterEnvSetup/node4.yml new file mode 100644 index 0000000000000000000000000000000000000000..c82a174cb883b14c885de7c5e8f19d98263b22b7 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/node4.yml @@ -0,0 +1,54 @@ +version: '3.7' + +services: + td2.0-node4: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - EXTRACTDIR=${DIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode4' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + volumes: + # bind data directory + - type: bind + source: /data/node4/data + target: /var/lib/taos + # bind log directory + - type: bind + source: /data/node4/log + target: /var/log/taos + # bind configuration + - type: bind + source: /data/node4/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: /data/node4/core + target: /coredump + - type: bind + source: /data + target: /root + hostname: tdnode4 + networks: + taos_update_net: + ipv4_address: 172.27.0.10 + command: taosd \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/node5.yml b/tests/pytest/cluster/clusterEnvSetup/node5.yml new file mode 100644 index 0000000000000000000000000000000000000000..2e37e47512430ac99244f6b2f0e2d309a2145edc --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/node5.yml @@ -0,0 +1,54 @@ +version: '3.7' + +services: + td2.0-node5: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - EXTRACTDIR=${DIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode5' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + volumes: + # bind data directory + - type: bind + source: /data/node5/data + target: /var/lib/taos + # bind log directory + - type: bind + source: /data/node5/log + target: /var/log/taos + # bind configuration + - type: bind + source: /data/node5/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: /data/node5/core + target: /coredump + - type: bind + source: /data + target: /root + hostname: tdnode5 + networks: + taos_update_net: + ipv4_address: 172.27.0.11 + command: taosd \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/taosdemoWrapper.py b/tests/pytest/cluster/clusterEnvSetup/taosdemoWrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..457dd4ee5aa5919951adbcea834d34cd367d3080 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/taosdemoWrapper.py @@ -0,0 +1,142 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import random +import argparse + +class taosdemoWrapper: + + def __init__(self, host, metadata, database, tables, threads, configDir, replica, + columnType, columnsPerTable, rowsPerTable, disorderRatio, disorderRange, charTypeLen): + self.host = host + self.metadata = metadata + self.database = database + self.tables = tables + self.threads = threads + self.configDir = configDir + self.replica = replica + self.columnType = columnType + self.columnsPerTable = columnsPerTable + self.rowsPerTable = rowsPerTable + self.disorderRatio = disorderRatio + self.disorderRange = disorderRange + self.charTypeLen = charTypeLen + + def run(self): + if self.metadata is None: + os.system("taosdemo -h %s -d %s -t %d -T %d -c %s -a %d -b %s -n %d -t %d -O %d -R %d -w %d -x -y" + % (self.host, self.database, self.tables, self.threads, self.configDir, self.replica, self.columnType, + self.rowsPerTable, self.disorderRatio, self.disorderRange, self.charTypeLen)) + else: + os.system("taosdemo -f %s" % self.metadata) + + +parser = argparse.ArgumentParser() +parser.add_argument( + '-H', + '--host-name', + action='store', + default='tdnode1', + type=str, + help='host name to be connected (default: tdnode1)') +parser.add_argument( + '-f', + '--metadata', + action='store', + default=None, + type=str, + help='The meta data to execution procedure, if use -f, all other options invalid, Default is NULL') +parser.add_argument( + '-d', + '--db-name', + action='store', + default='test', + type=str, + help='Database name to be created (default: test)') +parser.add_argument( + '-t', + '--num-of-tables', + action='store', + default=10, + type=int, + help='Number of tables (default: 10000)') +parser.add_argument( + '-T', + '--num-of-threads', + action='store', + default=10, + type=int, + help='Number of rest threads (default: 10)') +parser.add_argument( + '-c', + '--config-dir', + action='store', + default='/etc/taos/', + type=str, + help='Configuration directory. (Default is /etc/taos/)') +parser.add_argument( + '-a', + '--replica', + action='store', + default=100, + type=int, + help='Set the replica parameters of the database (default: 1, min: 1, max: 3)') +parser.add_argument( + '-b', + '--column-type', + action='store', + default='int', + type=str, + help='the data_type of columns (default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP)') +parser.add_argument( + '-l', + '--num-of-cols', + action='store', + default=10, + type=int, + help='The number of columns per record (default: 10)') +parser.add_argument( + '-n', + '--num-of-rows', + action='store', + default=1000, + type=int, + help='Number of subtales per stable (default: 1000)') +parser.add_argument( + '-O', + '--disorder-ratio', + action='store', + default=0, + type=int, + help=' (0: in order, > 0: disorder ratio, default: 0)') +parser.add_argument( + '-R', + '--disorder-range', + action='store', + default=0, + type=int, + help='Out of order datas range, ms (default: 1000)') +parser.add_argument( + '-w', + '--char-type-length', + action='store', + default=16, + type=int, + help='Out of order datas range, ms (default: 16)') + +args = parser.parse_args() +taosdemo = taosdemoWrapper(args.host_name, args.metadata, args.db_name, args.num_of_tables, + args.num_of_threads, args.config_dir, args.replica, args.column_type, args.num_of_cols, + args.num_of_rows, args.disorder_ratio, args.disorder_range, args.char_type_length) +taosdemo.run() diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py index dbda5657b6528402cc898762cccc78d74a196cd8..8a264270219232447a598768b45a37779edba698 100644 --- a/tests/pytest/cluster/clusterSetup.py +++ b/tests/pytest/cluster/clusterSetup.py @@ -11,15 +11,9 @@ # -*- coding: utf-8 -*- -import os -import sys -sys.path.insert(0, os.getcwd()) from fabric import Connection -from util.sql import * -from util.log import * -import taos import random -import threading +import time import logging class Node: @@ -76,6 +70,19 @@ class Node: print("remove taosd error for node %d " % self.index) logging.exception(e) + def forceStopOneTaosd(self): + try: + self.conn.run("kill -9 $(ps -ax|grep taosd|awk '{print $1}')") + except Exception as e: + print("kill taosd error on node%d " % self.index) + + def startOneTaosd(self): + try: + self.conn.run("nohup taosd -c /etc/taos/ > /dev/null 2>&1 &") + except Exception as e: + print("start taosd error on node%d " % self.index) + logging.exception(e) + def installTaosd(self, packagePath): self.conn.put(packagePath, self.homeDir) self.conn.cd(self.homeDir) @@ -122,100 +129,51 @@ class Node: class Nodes: def __init__(self): - self.node1 = Node(1, 'root', '52.151.60.239', 'node1', 'r', '/root/') - self.node2 = Node(2, 'root', '52.183.32.246', 'node1', 'r', '/root/') - self.node3 = Node(3, 'root', '51.143.46.79', 'node1', 'r', '/root/') - self.node4 = Node(4, 'root', '52.183.2.76', 'node1', 'r', '/root/') - self.node5 = Node(5, 'root', '13.66.225.87', 'node1', 'r', '/root/') + self.tdnodes = [] + self.tdnodes.append(Node(0, 'root', '52.143.103.7', 'node1', 'a', '/root/')) + self.tdnodes.append(Node(1, 'root', '52.250.48.222', 'node2', 'a', '/root/')) + self.tdnodes.append(Node(2, 'root', '51.141.167.23', 'node3', 'a', '/root/')) + self.tdnodes.append(Node(3, 'root', '52.247.207.173', 'node4', 'a', '/root/')) + self.tdnodes.append(Node(4, 'root', '51.141.166.100', 'node5', 'a', '/root/')) + + def stopOneNode(self, index): + self.tdnodes[index].forceStopOneTaosd() + + def startOneNode(self, index): + self.tdnodes[index].startOneTaosd() def stopAllTaosd(self): - self.node1.stopTaosd() - self.node2.stopTaosd() - self.node3.stopTaosd() - + for i in range(len(self.tdnodes)): + self.tdnodes[i].stopTaosd() + def startAllTaosd(self): - self.node1.startTaosd() - self.node2.startTaosd() - self.node3.startTaosd() + for i in range(len(self.tdnodes)): + self.tdnodes[i].startTaosd() def restartAllTaosd(self): - self.node1.restartTaosd() - self.node2.restartTaosd() - self.node3.restartTaosd() + for i in range(len(self.tdnodes)): + self.tdnodes[i].restartTaosd() def addConfigs(self, configKey, configValue): - self.node1.configTaosd(configKey, configValue) - self.node2.configTaosd(configKey, configValue) - self.node3.configTaosd(configKey, configValue) + for i in range(len(self.tdnodes)): + self.tdnodes[i].configTaosd(configKey, configValue) - def removeConfigs(self, configKey, configValue): - self.node1.removeTaosConfig(configKey, configValue) - self.node2.removeTaosConfig(configKey, configValue) - self.node3.removeTaosConfig(configKey, configValue) + def removeConfigs(self, configKey, configValue): + for i in range(len(self.tdnodes)): + self.tdnodes[i].removeTaosConfig(configKey, configValue) def removeAllDataFiles(self): - self.node1.removeData() - self.node2.removeData() - self.node3.removeData() - -class ClusterTest: - def __init__(self, hostName): - self.host = hostName - self.user = "root" - self.password = "taosdata" - self.config = "/etc/taos" - self.dbName = "mytest" - self.stbName = "meters" - self.numberOfThreads = 20 - self.numberOfTables = 10000 - self.numberOfRecords = 1000 - self.tbPrefix = "t" - self.ts = 1538548685000 - self.repeat = 1 - - def connectDB(self): - self.conn = taos.connect( - host=self.host, - user=self.user, - password=self.password, - config=self.config) - - def createSTable(self, replica): - cursor = self.conn.cursor() - tdLog.info("drop database if exists %s" % self.dbName) - cursor.execute("drop database if exists %s" % self.dbName) - tdLog.info("create database %s replica %d" % (self.dbName, replica)) - cursor.execute("create database %s replica %d" % (self.dbName, replica)) - tdLog.info("use %s" % self.dbName) - cursor.execute("use %s" % self.dbName) - tdLog.info("drop table if exists %s" % self.stbName) - cursor.execute("drop table if exists %s" % self.stbName) - tdLog.info("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) - cursor.execute("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) - cursor.close() - - def insertData(self, threadID): - print("Thread %d: starting" % threadID) - cursor = self.conn.cursor() - tablesPerThread = int(self.numberOfTables / self.numberOfThreads) - baseTableID = tablesPerThread * threadID - for i in range (tablesPerThread): - cursor.execute("create table %s%d using %s tags(%d)" % (self.tbPrefix, baseTableID + i, self.stbName, baseTableID + i)) - query = "insert into %s%d values" % (self.tbPrefix, baseTableID + i) - base = self.numberOfRecords * i - for j in range(self.numberOfRecords): - query += "(%d, %f, %d, %d)" % (self.ts + base + j, random.random(), random.randint(210, 230), random.randint(0, 10)) - cursor.execute(query) - cursor.close() - print("Thread %d: finishing" % threadID) - - def run(self): - threads = [] - tdLog.info("Inserting data") - for i in range(self.numberOfThreads): - thread = threading.Thread(target=self.insertData, args=(i,)) - threads.append(thread) - thread.start() - - for i in range(self.numberOfThreads): - threads[i].join() \ No newline at end of file + for i in range(len(self.tdnodes)): + self.tdnodes[i].removeData() + +# kill taosd randomly every 10 mins +nodes = Nodes() +loop = 0 +while True: + loop = loop + 1 + index = random.randint(0, 4) + print("loop: %d, kill taosd on node%d" %(loop, index)) + nodes.stopOneNode(index) + time.sleep(60) + nodes.startOneNode(index) + time.sleep(600) \ No newline at end of file diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 6f4258312d318bbfcc757153752d1fbebb205581..333c2a0a578f6f399e742b0967db0185a22b5fde 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -17,6 +17,9 @@ import json import time import random import requests +import argparse +import datetime +import string from requests.auth import HTTPBasicAuth func_list=['avg','count','twa','sum','stddev','leastsquares','min', 'max','first','last','top','bottom','percentile','apercentile', @@ -30,20 +33,39 @@ condition_list=[ 'fill(null)' ] -where_list = ['_c0>now-10d',' <50'," like \'%a%\'"] +where_list = ['_c0>now-10d',' <50','like',' is null'] class ConcurrentInquiry: - def __init__(self,n_Therads=25,r_Therads=25): + # def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test', + # stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5, + # stableNum = 2,subtableNum = 1000,insertRows = 100): + def __init__(self,ts,host,user,password,dbname, + stb_prefix,subtb_prefix,n_Therads,r_Therads,probabilities,loop, + stableNum ,subtableNum ,insertRows ,mix_table, replay): self.n_numOfTherads = n_Therads self.r_numOfTherads = r_Therads - self.ts=1500000001000 - self.dbname='test' + self.ts=ts + self.host = host + self.user = user + self.password = password + self.dbname=dbname + self.stb_prefix = stb_prefix + self.subtb_prefix = subtb_prefix self.stb_list=[] self.subtb_list=[] self.stb_stru_list=[] self.subtb_stru_list=[] self.stb_tag_list=[] self.subtb_tag_list=[] - + self.probabilities = [1-probabilities,probabilities] + self.ifjoin = [1,0] + self.loop = loop + self.stableNum = stableNum + self.subtableNum = subtableNum + self.insertRows = insertRows + self.mix_table = mix_table + self.max_ts = datetime.datetime.now() + self.min_ts = datetime.datetime.now() - datetime.timedelta(days=5) + self.replay = replay def SetThreadsNum(self,num): self.numOfTherads=num @@ -86,10 +108,18 @@ class ConcurrentInquiry: self.subtb_stru_list.append(tb) self.subtb_tag_list.append(tag) + def get_timespan(self,cl): #获取时间跨度(仅第一个超级表) + sql = 'select first(_c0),last(_c0) from ' + self.dbname + '.' + self.stb_list[0] + ';' + print(sql) + cl.execute(sql) + for data in cl: + self.max_ts = data[1] + self.min_ts = data[0] + def get_full(self): #获取所有的表、表结构 - host = "127.0.0.1" - user = "root" - password = "taosdata" + host = self.host + user = self.user + password = self.password conn = taos.connect( host, user, @@ -101,33 +131,53 @@ class ConcurrentInquiry: self.r_subtb_list(cl,i) self.r_stb_stru(cl) self.r_subtb_stru(cl) + self.get_timespan(cl) cl.close() conn.close() #query condition - def con_where(self,tlist): + def con_where(self,tlist,col_list,tag_list): l=[] for i in range(random.randint(0,len(tlist))): c = random.choice(where_list) if c == '_c0>now-10d': - l.append(c) + rdate = self.min_ts + (self.max_ts - self.min_ts)/10 * random.randint(-11,11) + conlist = ' _c0 ' + random.choice(['<','>','>=','<=','<>']) + "'" + str(rdate) + "'" + if self.random_pick(): + l.append(conlist) + else: l.append(c) + elif '<50' in c: + conlist = ' ' + random.choice(tlist) + random.choice(['<','>','>=','<=','<>']) + str(random.randrange(-100,100)) + l.append(conlist) + elif 'is null' in c: + conlist = ' ' + random.choice(tlist) + random.choice([' is null',' is not null']) + l.append(conlist) else: - l.append(random.choice(tlist)+c) + s_all = string.ascii_letters + conlist = ' ' + random.choice(tlist) + " like \'%" + random.choice(s_all) + "%\' " + l.append(conlist) return 'where '+random.choice([' and ',' or ']).join(l) - def con_interval(self,tlist): - return random.choice(['interval(10s)','interval(10d)','interval(1n)']) + def con_interval(self,tlist,col_list,tag_list): + interval = 'interval(' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')' + return interval - def con_limit(self,tlist): - return random.choice(['limit 10','limit 10 offset 10','slimit 10','slimit 10 offset 10','limit 10 slimit 10','limit 10 offset 5 slimit 5 soffset 10']) + def con_limit(self,tlist,col_list,tag_list): + rand1 = str(random.randint(0,1000)) + rand2 = str(random.randint(0,1000)) + return random.choice(['limit ' + rand1,'limit ' + rand1 + ' offset '+rand2, + ' slimit ' + rand1,' slimit ' + rand1 + ' offset ' + rand2,'limit '+rand1 + ' slimit '+ rand2, + 'limit '+ rand1 + ' offset' + rand2 + ' slimit '+ rand1 + ' soffset ' + rand2 ]) - def con_fill(self,tlist): + def con_fill(self,tlist,col_list,tag_list): return random.choice(['fill(null)','fill(prev)','fill(none)','fill(LINEAR)']) - def con_group(self,tlist): - return 'group by '+random.choice(tlist) + def con_group(self,tlist,col_list,tag_list): + rand_tag = random.randint(0,5) + rand_col = random.randint(0,1) + return 'group by '+','.join(random.sample(col_list,rand_col) + random.sample(tag_list,rand_tag)) - def con_order(self,tlist): + def con_order(self,tlist,col_list,tag_list): return 'order by '+random.choice(tlist) def gen_query_sql(self): #生成查询语句 @@ -157,32 +207,182 @@ class ConcurrentInquiry: random.shuffle(func_list) sel_col_list=[] col_rand=random.randint(0,len(col_list)) + loop = 0 for i,j in zip(col_list[0:col_rand],func_list): #决定每个被查询col的函数 + alias = ' as '+ 'taos%d ' % loop + loop += 1 + pick_func = '' if j == 'leastsquares': - sel_col_list.append(j+'('+i+',1,1)') + pick_func=j+'('+i+',1,1)' elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile': - sel_col_list.append(j+'('+i+',1)') + pick_func=j+'('+i+',1)' else: - sel_col_list.append(j+'('+i+')') - sql=sql+','.join(sel_col_list)+' from '+random.choice(self.stb_list+self.subtb_list)+' ' #select col & func + pick_func=j+'('+i+')' + if bool(random.getrandbits(1)): + pick_func+=alias + sel_col_list.append(pick_func) + if col_rand == 0: + sql = sql + '*' + else: + sql=sql+','.join(sel_col_list) #select col & func + if self.mix_table == 0: + sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' ' + elif self.mix_table == 1: + sql = sql + ' from '+random.choice(self.subtb_list)+' ' + else: + sql = sql + ' from '+random.choice(self.stb_list)+' ' con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill] sel_con=random.sample(con_func,random.randint(0,len(con_func))) sel_con_list=[] for i in sel_con: - sel_con_list.append(i(tlist)) #获取对应的条件函数 + sel_con_list.append(i(tlist,col_list,tag_list)) #获取对应的条件函数 sql+=' '.join(sel_con_list) # condition - print(sql) + #print(sql) return sql + def gen_query_join(self): #生成join查询语句 + tbname = [] + col_list = [] + tag_list = [] + col_intersection = [] + tag_intersection = [] + subtable = None + if self.mix_table == 0: + if bool(random.getrandbits(1)): + subtable = True + tbname = random.sample(self.subtb_list,2) + for i in tbname: + col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) + tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) + col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + else: + tbname = random.sample(self.stb_list,2) + for i in tbname: + col_list.append(self.stb_stru_list[self.stb_list.index(i)]) + tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) + col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + elif self.mix_table == 1: + subtable = True + tbname = random.sample(self.subtb_list,2) + for i in tbname: + col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) + tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) + col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + else: + tbname = random.sample(self.stb_list,2) + for i in tbname: + col_list.append(self.stb_stru_list[self.stb_list.index(i)]) + tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) + col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + con_rand=random.randint(0,len(condition_list)) + col_rand=random.randint(0,len(col_list)) + tag_rand=random.randint(0,len(tag_list)) + sql='select ' #select + + sel_col_tag=[] + col_rand=random.randint(0,len(col_list)) + if bool(random.getrandbits(1)): + sql += '*' + else: + sel_col_tag.append('t1.' + str(random.choice(col_list[0] + tag_list[0]))) + sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1]))) + sel_col_list = [] + random.shuffle(func_list) + if self.random_pick(): + loop = 0 + for i,j in zip(sel_col_tag,func_list): #决定每个被查询col的函数 + alias = ' as '+ 'taos%d ' % loop + loop += 1 + pick_func = '' + if j == 'leastsquares': + pick_func=j+'('+i+',1,1)' + elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile': + pick_func=j+'('+i+',1)' + else: + pick_func=j+'('+i+')' + if bool(random.getrandbits(1)): + pick_func+=alias + sel_col_list.append(pick_func) + sql += ','.join(sel_col_list) + else: + sql += ','.join(sel_col_tag) + + sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func + join_section = None + temp = None + if subtable: + temp = random.choices(col_intersection) + join_section = temp.pop() + sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) + else: + temp = random.choices(col_intersection+tag_intersection) + join_section = temp.pop() + sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) + return sql + + def random_pick(self): + x = random.uniform(0,1) + cumulative_probability = 0.0 + for item, item_probability in zip(self.ifjoin, self.probabilities): + cumulative_probability += item_probability + if x < cumulative_probability:break + return item + + def gen_data(self): + stableNum = self.stableNum + subtableNum = self.subtableNum + insertRows = self.insertRows + t0 = self.ts + host = self.host + user = self.user + password = self.password + conn = taos.connect( + host, + user, + password, + ) + cl = conn.cursor() + cl.execute("drop database if exists %s;" %self.dbname) + cl.execute("create database if not exists %s;" %self.dbname) + cl.execute("use %s" % self.dbname) + for k in range(stableNum): + sql="create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20),c11 int unsigned,c12 smallint unsigned,c13 tinyint unsigned,c14 bigint unsigned) \ + tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20), t11 int unsigned , t12 smallint unsigned , t13 tinyint unsigned , t14 bigint unsigned)" % (self.stb_prefix+str(k)) + cl.execute(sql) + for j in range(subtableNum): + if j % 100 == 0: + sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \ + (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k)) + else: + sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \ + (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167) + print(sql) + cl.execute(sql) + for i in range(insertRows): + if i % 100 == 0 : + ret = cl.execute( + "insert into %s values (%d , NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % + (self.subtb_prefix+str(k)+'_'+str(j), t0+i)) + else: + ret = cl.execute( + "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % + (self.subtb_prefix+str(k)+'_'+str(j), t0+i, i%100, i/2.0, i%41, i%51, i%53, i*1.0, i%2,'taos'+str(i),'涛思'+str(i), i%43, i%23 , i%17 , i%3167)) + cl.close() + conn.close() + def rest_query(self,sql): #rest 接口 - host = "127.0.0.1" - user = "root" - password = "taosdata" + host = self.host + user = self.user + password = self.password port =6041 url = "http://{}:{}/rest/sql".format(host, port ) try: r = requests.post(url, - data = 'use test', + data = 'use %s' % self.dbname, auth = HTTPBasicAuth('root', 'taosdata')) r = requests.post(url, data = sql, @@ -210,74 +410,291 @@ class ConcurrentInquiry: nRows = rj['rows'] if ('rows' in rj) else 0 return nRows + def query_thread_n(self,threadID): #使用原生python接口查询 - host = "127.0.0.1" - user = "root" - password = "taosdata" + host = self.host + user = self.user + password = self.password conn = taos.connect( host, user, password, ) cl = conn.cursor() - cl.execute("use test;") - + cl.execute("use %s;" % self.dbname) + fo = open('bak_sql_n_%d'%threadID,'w+') print("Thread %d: starting" % threadID) - - while True: + loop = self.loop + while loop: try: - sql=self.gen_query_sql() + if self.random_pick(): + sql=self.gen_query_sql() + else: + sql=self.gen_query_join() print("sql is ",sql) + fo.write(sql+'\n') start = time.time() cl.execute(sql) cl.fetchall() end = time.time() print("time cost :",end-start) except Exception as e: + print('-'*40) print( - "Failure thread%d, sql: %s,exception: %s" % + "Failure thread%d, sql: %s \nexception: %s" % (threadID, str(sql),str(e))) - #exit(-1) - - + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + loop -= 1 + if loop == 0: break + fo.close() + cl.close() + conn.close() print("Thread %d: finishing" % threadID) + + def query_thread_nr(self,threadID): #使用原生python接口进行重放 + host = self.host + user = self.user + password = self.password + conn = taos.connect( + host, + user, + password, + ) + cl = conn.cursor() + cl.execute("use %s;" % self.dbname) + replay_sql = [] + with open('bak_sql_n_%d'%threadID,'r') as f: + replay_sql = f.readlines() + print("Replay Thread %d: starting" % threadID) + for sql in replay_sql: + try: + print("sql is ",sql) + start = time.time() + cl.execute(sql) + cl.fetchall() + end = time.time() + print("time cost :",end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql),str(e))) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + cl.close() + conn.close() + print("Replay Thread %d: finishing" % threadID) def query_thread_r(self,threadID): #使用rest接口查询 print("Thread %d: starting" % threadID) - while True: - try: + fo = open('bak_sql_r_%d'%threadID,'w+') + loop = self.loop + while loop: + try: + if self.random_pick(): sql=self.gen_query_sql() - print("sql is ",sql) - start = time.time() - self.rest_query(sql) - end = time.time() - print("time cost :",end-start) - except Exception as e: - print( - "Failure thread%d, sql: %s,exception: %s" % - (threadID, str(sql),str(e))) - #exit(-1) - - - print("Thread %d: finishing" % threadID) + else: + sql=self.gen_query_join() + print("sql is ",sql) + fo.write(sql+'\n') + start = time.time() + self.rest_query(sql) + end = time.time() + print("time cost :",end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql),str(e))) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + loop -= 1 + if loop == 0: break + fo.close() + print("Thread %d: finishing" % threadID) + + def query_thread_rr(self,threadID): #使用rest接口重放 + print("Replay Thread %d: starting" % threadID) + replay_sql = [] + with open('bak_sql_r_%d'%threadID,'r') as f: + replay_sql = f.readlines() + + for sql in replay_sql: + try: + print("sql is ",sql) + start = time.time() + self.rest_query(sql) + end = time.time() + print("time cost :",end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql),str(e))) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + print("Replay Thread %d: finishing" % threadID) def run(self): print(self.n_numOfTherads,self.r_numOfTherads) threads = [] - for i in range(self.n_numOfTherads): - thread = threading.Thread(target=self.query_thread_n, args=(i,)) - threads.append(thread) - thread.start() - for i in range(self.r_numOfTherads): - # for i in range(1): - thread = threading.Thread(target=self.query_thread_r, args=(i,)) - threads.append(thread) - thread.start() -if len(sys.argv)>1: - q = ConcurrentInquiry(n_Therads=sys.argv[1],r_Therads=sys.argv[2]) -else: - q = ConcurrentInquiry() + if self.replay: #whether replay + for i in range(self.n_numOfTherads): + thread = threading.Thread(target=self.query_thread_nr, args=(i,)) + threads.append(thread) + thread.start() + for i in range(self.r_numOfTherads): + thread = threading.Thread(target=self.query_thread_rr, args=(i,)) + threads.append(thread) + thread.start() + else: + for i in range(self.n_numOfTherads): + thread = threading.Thread(target=self.query_thread_n, args=(i,)) + threads.append(thread) + thread.start() + for i in range(self.r_numOfTherads): + thread = threading.Thread(target=self.query_thread_r, args=(i,)) + threads.append(thread) + thread.start() + +parser = argparse.ArgumentParser() +parser.add_argument( + '-H', + '--host-name', + action='store', + default='127.0.0.1', + type=str, + help='host name to be connected (default: 127.0.0.1)') +parser.add_argument( + '-S', + '--ts', + action='store', + default=1500000000000, + type=int, + help='insert data from timestamp (default: 1500000000000)') +parser.add_argument( + '-d', + '--db-name', + action='store', + default='test', + type=str, + help='Database name to be created (default: test)') +parser.add_argument( + '-t', + '--number-of-native-threads', + action='store', + default=10, + type=int, + help='Number of native threads (default: 10)') +parser.add_argument( + '-T', + '--number-of-rest-threads', + action='store', + default=10, + type=int, + help='Number of rest threads (default: 10)') +parser.add_argument( + '-r', + '--number-of-records', + action='store', + default=100, + type=int, + help='Number of record to be created for each table (default: 100)') +parser.add_argument( + '-c', + '--create-table', + action='store', + default='0', + type=int, + help='whether gen data (default: 0)') +parser.add_argument( + '-p', + '--subtb-name-prefix', + action='store', + default='t', + type=str, + help='subtable-name-prefix (default: t)') +parser.add_argument( + '-P', + '--stb-name-prefix', + action='store', + default='st', + type=str, + help='stable-name-prefix (default: st)') +parser.add_argument( + '-b', + '--probabilities', + action='store', + default='0.05', + type=float, + help='probabilities of join (default: 0.05)') +parser.add_argument( + '-l', + '--loop-per-thread', + action='store', + default='100', + type=int, + help='loop per thread (default: 100)') +parser.add_argument( + '-u', + '--user', + action='store', + default='root', + type=str, + help='user name') +parser.add_argument( + '-w', + '--password', + action='store', + default='root', + type=str, + help='user name') +parser.add_argument( + '-n', + '--number-of-tables', + action='store', + default=1000, + type=int, + help='Number of subtales per stable (default: 1000)') +parser.add_argument( + '-N', + '--number-of-stables', + action='store', + default=2, + type=int, + help='Number of stables (default: 2)') +parser.add_argument( + '-m', + '--mix-stable-subtable', + action='store', + default=0, + type=int, + help='0:stable & substable ,1:subtable ,2:stable (default: 0)') +parser.add_argument( + '-R', + '--replay', + action='store', + default=0, + type=int, + help='0:not replay ,1:replay (default: 0)') + +args = parser.parse_args() +q = ConcurrentInquiry( + args.ts,args.host_name,args.user,args.password,args.db_name, + args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads, + args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records, + args.mix_stable_subtable, args.replay ) + +if args.create_table: + q.gen_data() q.get_full() + #q.gen_query_sql() q.run() + diff --git a/tests/pytest/concurrent_inquiry.sh b/tests/pytest/concurrent_inquiry.sh new file mode 100755 index 0000000000000000000000000000000000000000..f426fbbcec3070789209eb787dba61d95571f0e5 --- /dev/null +++ b/tests/pytest/concurrent_inquiry.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# This is the script for us to try to cause the TDengine server or client to crash +# +# PREPARATION +# +# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree +# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory +# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg +# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg +# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above +# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils +# +# RUNNING THIS SCRIPT +# +# This script assumes the source code directory is intact, and that the binaries has been built in the +# build/ directory, as such, will will load the Python libraries in the directory tree, and also load +# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env +# variables below. +# +# Running the script is simple, no parameter is needed (for now, but will change in the future). +# +# Happy Crashing... + + +# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory +EXEC_DIR=`dirname "$0"` +if [[ $EXEC_DIR != "." ]] +then + echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)" + exit -1 +fi + +CURR_DIR=`pwd` +IN_TDINTERNAL="community" +if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + TAOS_DIR=$CURR_DIR/../../.. + TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6,7|rev`/lib +else + TAOS_DIR=$CURR_DIR/../.. + TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib +fi + +# Now getting ready to execute Python +# The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk +PYTHON_EXEC=python3.8 + +# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. +export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd) + +# Then let us set up the library path so that our compiled SO file can be loaded by Python +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR + +# Now we are all let, and let's see if we can find a crash. Note we pass all params +CONCURRENT_INQUIRY=concurrent_inquiry.py +if [[ $1 == '--valgrind' ]]; then + shift + export PYTHONMALLOC=malloc + VALGRIND_OUT=valgrind.out + VALGRIND_ERR=valgrind.err + # How to generate valgrind suppression file: https://stackoverflow.com/questions/17159578/generating-suppressions-for-memory-leaks + # valgrind --leak-check=full --gen-suppressions=all --log-fd=9 python3.8 ./concurrent_inquiry.py $@ 9>>memcheck.log + echo Executing under VALGRIND, with STDOUT/ERR going to $VALGRIND_OUT and $VALGRIND_ERR, please watch them from a different terminal. + valgrind \ + --leak-check=yes \ + --suppressions=crash_gen/valgrind_taos.supp \ + $PYTHON_EXEC \ + $CONCURRENT_INQUIRY $@ > $VALGRIND_OUT 2> $VALGRIND_ERR +elif [[ $1 == '--helgrind' ]]; then + shift + HELGRIND_OUT=helgrind.out + HELGRIND_ERR=helgrind.err + valgrind \ + --tool=helgrind \ + $PYTHON_EXEC \ + $CONCURRENT_INQUIRY $@ > $HELGRIND_OUT 2> $HELGRIND_ERR +else + $PYTHON_EXEC $CONCURRENT_INQUIRY $@ +fi + diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index e2ce4b26fa2fa8cc9c601218c1bf19d838b57c46..309c0df9108f340bf96d73529ccf8bb49c1c9692 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -354,10 +354,11 @@ class ThreadCoordinator: # end, and maybe signal them to stop if isinstance(err, CrashGenError): # our own transition failure Logging.info("State transition error") + # TODO: saw an error here once, let's print out stack info for err? traceback.print_stack() transitionFailed = True self._te = None # Not running any more - self._execStats.registerFailure("State transition error") + self._execStats.registerFailure("State transition error: {}".format(err)) else: raise # return transitionFailed # Why did we have this??!! @@ -882,8 +883,12 @@ class StateMechine: self._stateWeights = [1, 2, 10, 40] def init(self, dbc: DbConn): # late initailization, don't save the dbConn - self._curState = self._findCurrentState(dbc) # starting state - Logging.debug("Found Starting State: {}".format(self._curState)) + try: + self._curState = self._findCurrentState(dbc) # starting state + except taos.error.ProgrammingError as err: + Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) + traceback.print_stack() + raise # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -951,6 +956,8 @@ class StateMechine: # We transition the system to a new state by examining the current state itself def transition(self, tasks, dbc: DbConn): + global gSvcMgr + if (len(tasks) == 0): # before 1st step, or otherwise empty Logging.debug("[STT] Starting State: {}".format(self._curState)) return # do nothing @@ -1276,6 +1283,7 @@ class Task(): 0x510, # vnode not in ready state 0x14, # db not ready, errno changed 0x600, # Invalid table ID, why? + 0x218, # Table does not exist 1000 # REST catch-all error ]: return True # These are the ALWAYS-ACCEPTABLE ones @@ -2224,22 +2232,25 @@ class ClientManager: if svcMgr: # gConfig.auto_start_service: svcMgr.stopTaosServices() svcMgr = None - # Print exec status, etc., AFTER showing messages from the server - self.conclude() - # print("TC failed (2) = {}".format(self.tc.isFailed())) - # Linux return code: ref https://shapeshed.com/unix-exit-codes/ - ret = 1 if self.tc.isFailed() else 0 - self.tc.cleanup() + # Release global variables gConfig = None gSvcMgr = None logger = None + + thPool = None + dbManager.cleanUp() # destructor wouldn't run in time + dbManager = None + # Print exec status, etc., AFTER showing messages from the server + self.conclude() + # print("TC failed (2) = {}".format(self.tc.isFailed())) + # Linux return code: ref https://shapeshed.com/unix-exit-codes/ + ret = 1 if self.tc.isFailed() else 0 + self.tc.cleanup() # Release variables here self.tc = None - thPool = None - dbManager = None gc.collect() # force garbage collection # h = hpy() @@ -2366,7 +2377,7 @@ class MainExec: '-n', '--dynamic-db-table-names', action='store_true', - help='Use non-fixed names for dbs/tables, useful for multi-instance executions (default: false)') + help='Use non-fixed names for dbs/tables, for -b, useful for multi-instance executions (default: false)') parser.add_argument( '-o', '--num-dnodes', diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 855e18be55fd5ae0d88865644a2ebfc836b7ccf3..e38692dbe1e5c33ffe162015e3e60630fd51fa38 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -15,6 +15,7 @@ from util.log import * from .misc import Logging, CrashGenError, Helper, Dice import os import datetime +import traceback # from .service_manager import TdeInstance class DbConn: @@ -349,6 +350,7 @@ class DbConnNative(DbConn): def execute(self, sql): if (not self.isOpen): + traceback.print_stack() raise CrashGenError( "Cannot exec SQL unless db connection is open", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) @@ -361,6 +363,7 @@ class DbConnNative(DbConn): def query(self, sql): # return rows affected if (not self.isOpen): + traceback.print_stack() raise CrashGenError( "Cannot query database until connection is open, restarting?", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) @@ -394,6 +397,7 @@ class DbManager(): cType == 'native') else DbConn.createRest(dbTarget) try: self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected + Logging.debug("DbManager opened DB connection...") except taos.error.ProgrammingError as err: # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) if (err.msg == 'client disconnected'): # cannot open DB connection @@ -412,6 +416,10 @@ class DbManager(): # Moved to Database() # self._stateMachine = StateMechine(self._dbConn) + def __del__(self): + ''' Release the underlying DB connection upon deletion of DbManager ''' + self.cleanUp() + def getDbConn(self): return self._dbConn @@ -438,5 +446,8 @@ class DbManager(): return "table_{}".format(tblNum) def cleanUp(self): - self._dbConn.close() + if self._dbConn: + self._dbConn.close() + self._dbConn = None + Logging.debug("DbManager closed DB connection...") diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index a374ed943b41b52c2ccbee0825bca3080ed43ea9..6ea5691ce223eb1c14214d4b11c47cf85e29c795 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -2,6 +2,7 @@ import threading import random import logging import os +import sys import taos @@ -53,7 +54,7 @@ class Logging: # global misc.logger _logger = logging.getLogger('CrashGen') # real logger _logger.addFilter(LoggingFilter()) - ch = logging.StreamHandler() + ch = logging.StreamHandler(sys.stdout) # Ref: https://stackoverflow.com/questions/14058453/making-python-loggers-output-all-messages-to-stdout-in-addition-to-log-file _logger.addHandler(ch) # Logging adapter, to be used as a logger diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py index fd12284b9d7782ac7df89c37fcb653ca3bebe82b..de2d9b0780f3654424d8e077a4dda9ac5ac0ed50 100644 --- a/tests/pytest/crash_gen_bootstrap.py +++ b/tests/pytest/crash_gen_bootstrap.py @@ -19,5 +19,5 @@ if __name__ == "__main__": mExec.init() exitCode = mExec.run() - print("Exiting with code: {}".format(exitCode)) + print("\nCrash_Gen is now exiting with status code: {}".format(exitCode)) sys.exit(exitCode) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 42af09e7ebf9b6a2945e578f707ba020af274466..14f2b6e07e39322a3ce62f6176034fd2fba9edbc 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -3,12 +3,16 @@ ulimit -c unlimited python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/int.py +python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/float.py python3 ./test.py -f insert/bigint.py +python3 ./test.py -f insert/unsignedBigint.py python3 ./test.py -f insert/bool.py python3 ./test.py -f insert/double.py python3 ./test.py -f insert/smallint.py +python3 ./test.py -f insert/unsignedSmallint.py python3 ./test.py -f insert/tinyint.py +python3 ./test.py -f insert/unsignedTinyint.py python3 ./test.py -f insert/date.py python3 ./test.py -f insert/binary.py python3 ./test.py -f insert/nchar.py @@ -19,14 +23,24 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py -python3 ./test.py -f insert/before_1970.py +#python3 ./test.py -f insert/before_1970.py +python3 ./test.py -f insert/metadataUpdate.py +python3 bug2265.py +python3 ./test.py -f insert/boundary2.py +#table python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py python3 ./test.py -f table/create_sensitive.py #python3 ./test.py -f table/tablename-boundary.py +python3 ./test.py -f table/max_table_length.py +python3 ./test.py -f table/alter_column.py +python3 ./test.py -f table/boundary.py +python3 ./test.py -f table/create.py +python3 ./test.py -f table/del_stable.py + # tag python3 ./test.py -f tag_lite/filter.py @@ -57,7 +71,7 @@ python3 ./test.py -f tag_lite/int.py python3 ./test.py -f tag_lite/set.py python3 ./test.py -f tag_lite/smallint.py python3 ./test.py -f tag_lite/tinyint.py - +python3 ./test.py -f tag_lite/alter_tag.py #python3 ./test.py -f dbmgmt/database-name-boundary.py python3 ./test.py -f import_merge/importBlock1HO.py @@ -135,15 +149,13 @@ python3 ./test.py -f user/pass_len.py # stable python3 ./test.py -f stable/query_after_reset.py -# table -python3 ./test.py -f table/del_stable.py - #query python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterCombo.py python3 ./test.py -f query/queryNormal.py python3 ./test.py -f query/queryError.py python3 ./test.py -f query/filterAllIntTypes.py +python3 ./test.py -f query/filterAllUnsignedIntTypes.py python3 ./test.py -f query/filterFloatAndDouble.py python3 ./test.py -f query/filterOtherTypes.py python3 ./test.py -f query/querySort.py @@ -161,24 +173,39 @@ python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug2218.py python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2118.py python3 ./test.py -f query/bug2143.py -python3 ./test.py -f query/sliding.py +python3 ./test.py -f query/sliding.py +python3 ./test.py -f query/unionAllTest.py +python3 ./test.py -f query/bug2281.py +python3 ./test.py -f query/bug2119.py +python3 ./test.py -f query/isNullTest.py +python3 ./test.py -f query/queryWithTaosdKilled.py +python3 ./test.py -f query/floatCompare.py +python3 ./test.py -f query/queryGroupbySort.py #stream python3 ./test.py -f stream/metric_1.py +python3 ./test.py -f stream/metric_n.py python3 ./test.py -f stream/new.py python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py #python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py +python3 ./test.py -f stream/sys.py +python3 ./test.py -f stream/table_1.py +python3 ./test.py -f stream/table_n.py #alter table python3 ./test.py -f alter/alter_table_crash.py +python3 ./test.py -f alter/alter_table.py +python3 ./test.py -f alter/alter_debugFlag.py # client python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py +python3 ./test.py -f client/noConnectionErrorTest.py # Misc python3 testCompress.py @@ -202,17 +229,22 @@ python3 ./test.py -f functions/function_spread.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 -#python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa_test2.py +python3 ./test.py -f functions/all_null_value.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py python3 client/twoClients.py -python3 test.py -f query/queryInterval.py -python3 test.py -f query/queryFillTest.py +python3 ./test.py -f query/queryInterval.py +python3 ./test.py -f query/queryFillTest.py +python3 ./test.py -f query/last_row_cache.py +python3 ./test.py -f query/last_cache.py # tools python3 test.py -f tools/taosdemoTest.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/lowaTest.py +python3 test.py -f tools/taosdemoTest2.py # subscribe python3 test.py -f subscribe/singlemeter.py @@ -235,4 +267,7 @@ python3 ./test.py -f update/merge_commit_last.py python3 ./test.py -f update/bug_td2279.py # wal -python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file +python3 ./test.py -f wal/addOldWalTest.py + +# account +python3 ./test.py -f account/account_create.py \ No newline at end of file diff --git a/tests/pytest/functions/all_null_value.py b/tests/pytest/functions/all_null_value.py new file mode 100644 index 0000000000000000000000000000000000000000..64791831107de4720ea43f26c250ebfb3da7195d --- /dev/null +++ b/tests/pytest/functions/all_null_value.py @@ -0,0 +1,90 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 int, c2 int)") + for i in range(self.rowNum): + tdSql.execute("insert into st values(%d, null, null)" % (self.ts + i)) + + tdSql.query("select avg(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select max(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select min(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select bottom(c1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select top(c1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select diff(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select first(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select last(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select last_row(c1) from st") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select count(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select leastsquares(c1, 1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select c1 + c2 from st") + tdSql.checkRows(10) + + tdSql.query("select spread(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select stddev(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select sum(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select twa(c1) from st") + tdSql.checkRows(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_avg.py b/tests/pytest/functions/function_avg.py index 9481550ba3f9d912b734d19b7c794c495fffdecd..c8a1cca77b1568b760b0208e0fa71583ae2e3645 100644 --- a/tests/pytest/functions/function_avg.py +++ b/tests/pytest/functions/function_avg.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,7 +63,15 @@ class TDTestCase: tdSql.query("select avg(col5) from test") tdSql.checkData(0, 0, np.average(floatData)) tdSql.query("select avg(col6) from test") - tdSql.checkData(0, 0, np.average(floatData)) + tdSql.checkData(0, 0, np.average(floatData)) + tdSql.query("select avg(col11) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col12) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col13) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col14) from test") + tdSql.checkData(0, 0, np.average(intData)) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_avg_restart.py b/tests/pytest/functions/function_avg_restart.py index 56b99cdf910869b5bc4f5bd93da208943b1b026e..60f3ba7d4e2708053425b4866e072516b5ff961c 100644 --- a/tests/pytest/functions/function_avg_restart.py +++ b/tests/pytest/functions/function_avg_restart.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - #tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + #tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,7 +63,15 @@ class TDTestCase: tdSql.query("select avg(col5) from test") tdSql.checkData(0, 0, np.average(floatData)) tdSql.query("select avg(col6) from test") - tdSql.checkData(0, 0, np.average(floatData)) + tdSql.checkData(0, 0, np.average(floatData)) + tdSql.query("select avg(col11) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col12) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col13) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col14) from test") + tdSql.checkData(0, 0, np.average(intData)) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index 4074166f927acbbcbb5899a692ff87f3c187f92f..abb9ac48e70ac876741c3da2a9f223d537c8644b 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # bottom verifacation tdSql.error("select bottom(ts, 10) from test") @@ -84,7 +84,43 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 0.1) tdSql.checkData(1, 1, 1.1) + + tdSql.query("select bottom(col11, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col12, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col13, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col14, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + #TD-2457 bottom + interval + order by + tdSql.error('select top(col2,1) from test interval(1y) order by col2;') + + #TD-2563 top + super_table + interval + tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") + tdSql.execute("create table t1 using meters tags (1)") + sql = 'insert into t1 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i , i % 47) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t1 values ' + tdSql.execute(sql) + tdSql.query('select bottom(c,1) from meters interval(10a)') + tdSql.checkData(0,1,0) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/functions/function_bottom_restart.py b/tests/pytest/functions/function_bottom_restart.py index 74eb044645ffa6a54e286dd11d022672575a38db..b4d987975c3660e2ea875a03f3406eaebcafea6d 100644 --- a/tests/pytest/functions/function_bottom_restart.py +++ b/tests/pytest/functions/function_bottom_restart.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.execute("use db") #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") #for i in range(self.rowNum): - # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # bottom verifacation tdSql.error("select bottom(ts, 10) from test") @@ -75,6 +75,26 @@ class TDTestCase: tdSql.checkData(0, 1, 1) tdSql.checkData(1, 1, 2) + tdSql.query("select bottom(col11, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col12, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col13, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col14, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + tdSql.query("select bottom(col5, 2) from test") tdSql.checkRows(2) tdSql.checkData(0, 1, 0.1) diff --git a/tests/pytest/functions/function_count.py b/tests/pytest/functions/function_count.py index 4795a6b2dfe72bb9fa747b550be9f56c0c255d88..9812473d6496a94447734f1f3507ac806e392294 100644 --- a/tests/pytest/functions/function_count.py +++ b/tests/pytest/functions/function_count.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # Count verifacation tdSql.query("select count(*) from test") @@ -62,11 +62,20 @@ class TDTestCase: tdSql.query("select count(col9) from test") tdSql.checkData(0, 0, 10) + tdSql.query("select count(col11) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col12) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col13) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col14) from test") + tdSql.checkData(0, 0, 10) + tdSql.execute("alter table test add column col10 int") tdSql.query("select count(col10) from test") tdSql.checkRows(0) - tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' 1)") + tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' , 1, 1, 1, 1, 1)") tdSql.query("select count(col10) from test") tdSql.checkData(0, 0, 1) diff --git a/tests/pytest/functions/function_count_restart.py b/tests/pytest/functions/function_count_restart.py index 5eabb47d952a1a461b6fd5c976f3f2a8c294222b..565d85f4c0712b436e2dc24335aa13f60d3c87dd 100644 --- a/tests/pytest/functions/function_count_restart.py +++ b/tests/pytest/functions/function_count_restart.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.execute("use db") #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") #for i in range(self.rowNum): - # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # Count verifacation tdSql.query("select count(*) from test") @@ -62,6 +62,15 @@ class TDTestCase: tdSql.query("select count(col9) from test") tdSql.checkData(0, 0, 11) + tdSql.query("select count(col11) from test") + tdSql.checkData(0, 0, 11) + tdSql.query("select count(col12) from test") + tdSql.checkData(0, 0, 11) + tdSql.query("select count(col13) from test") + tdSql.checkData(0, 0, 11) + tdSql.query("select count(col14) from test") + tdSql.checkData(0, 0, 11) + #tdSql.execute("alter table test add column col10 int") #tdSql.query("select count(col10) from test") #tdSql.checkRows(0) diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py index b6f496d7d68ebaa0f19dfc947d7745802198a3dc..46ea3a86deee75d26f0b635ae0d3b2c5673fb6da 100644 --- a/tests/pytest/functions/function_diff.py +++ b/tests/pytest/functions/function_diff.py @@ -31,9 +31,9 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") - tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ', 0, 0, 0, 0)" % (self.ts - 1)) # diff verifacation tdSql.query("select diff(col1) from test1") @@ -55,8 +55,8 @@ class TDTestCase: tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.error("select diff(ts) from test") tdSql.error("select diff(ts) from test1") @@ -72,7 +72,16 @@ class TDTestCase: tdSql.error("select diff(col8) from test1") tdSql.error("select diff(col9) from test") tdSql.error("select diff(col9) from test1") - + tdSql.error("select diff(col11) from test1") + tdSql.error("select diff(col12) from test1") + tdSql.error("select diff(col13) from test1") + tdSql.error("select diff(col14) from test1") + tdSql.error("select diff(col11) from test") + tdSql.error("select diff(col12) from test") + tdSql.error("select diff(col13) from test") + tdSql.error("select diff(col14) from test") + + tdSql.query("select diff(col1) from test1") tdSql.checkRows(10) diff --git a/tests/pytest/functions/function_diff_restart.py b/tests/pytest/functions/function_diff_restart.py index 870ee03fc94cbd8dbf5e858726cd52b09010a884..e03b892477eba46c007163049d91012af236e730 100644 --- a/tests/pytest/functions/function_diff_restart.py +++ b/tests/pytest/functions/function_diff_restart.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.execute("use db") #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") #tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) @@ -55,8 +55,8 @@ class TDTestCase: #tdSql.checkRows(0) #for i in range(self.rowNum): - # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.error("select diff(ts) from test") tdSql.error("select diff(ts) from test1") @@ -71,7 +71,14 @@ class TDTestCase: tdSql.error("select diff(col8) from test") tdSql.error("select diff(col8) from test1") tdSql.error("select diff(col9) from test") - tdSql.error("select diff(col9) from test1") + tdSql.error("select diff(col11) from test1") + tdSql.error("select diff(col12) from test1") + tdSql.error("select diff(col13) from test1") + tdSql.error("select diff(col14) from test1") + tdSql.error("select diff(col11) from test") + tdSql.error("select diff(col12) from test") + tdSql.error("select diff(col13) from test") + tdSql.error("select diff(col14) from test") tdSql.query("select diff(col1) from test1") tdSql.checkRows(10) diff --git a/tests/pytest/functions/function_first.py b/tests/pytest/functions/function_first.py index 2b78bd33d8bdb0bcd6775215b1cf0b1e398f2928..f1a916b168271e67d0001c5d4444966f8c07a2d1 100644 --- a/tests/pytest/functions/function_first.py +++ b/tests/pytest/functions/function_first.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) @@ -52,6 +52,18 @@ class TDTestCase: tdSql.query("select first(col4) from test1") tdSql.checkRows(0) + tdSql.query("select first(col11) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col12) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col13) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col14) from test1") + tdSql.checkRows(0) + tdSql.query("select first(col5) from test1") tdSql.checkRows(0) @@ -68,8 +80,8 @@ class TDTestCase: tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select first(*) from test1") tdSql.checkRows(1) @@ -91,6 +103,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 1) + tdSql.query("select first(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.query("select first(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 0.1) @@ -110,6 +138,10 @@ class TDTestCase: tdSql.query("select first(col9) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, '涛思数据1') + + # TD-2607 first,last + where none exist condition + interval + tdSql.query("select first(*),last(*) from test1 where ts < 23 interval(1s)") + tdSql.checkRows(0) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_last.py b/tests/pytest/functions/function_last.py index 337c18a28c2cd99d556e5b017377edc56671c490..991ac96a800803440a2e662c163622af95c556e3 100644 --- a/tests/pytest/functions/function_last.py +++ b/tests/pytest/functions/function_last.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) @@ -52,6 +52,18 @@ class TDTestCase: tdSql.query("select last(col4) from test1") tdSql.checkRows(0) + tdSql.query("select last(col11) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col12) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col13) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col14) from test1") + tdSql.checkRows(0) + tdSql.query("select last(col5) from test1") tdSql.checkRows(0) @@ -68,8 +80,8 @@ class TDTestCase: tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select last(*) from test1") tdSql.checkRows(1) @@ -91,6 +103,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_last_row.py b/tests/pytest/functions/function_last_row.py index ea5cf661eb9e70e2ed06ebd712b2f65dfa62beeb..d075eeeca55c7e6575ab9bd526ddb6177bf61791 100644 --- a/tests/pytest/functions/function_last_row.py +++ b/tests/pytest/functions/function_last_row.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) @@ -56,6 +56,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, None) + tdSql.query("select last_row(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.query("select last_row(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, None) @@ -77,8 +93,8 @@ class TDTestCase: tdSql.checkData(0, 0, None) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select last_row(*) from test1") tdSql.checkRows(1) @@ -100,6 +116,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select last_row(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last_row(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_leastsquares.py b/tests/pytest/functions/function_leastsquares.py index 0ef0a2b78612dca604f7a7a7f70841c917376a17..3b2a8486d5e6d8bbc2464b8ee391181e10e6a285 100644 --- a/tests/pytest/functions/function_leastsquares.py +++ b/tests/pytest/functions/function_leastsquares.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # leastsquares verifacation tdSql.error("select leastsquares(ts, 1, 1) from test1") @@ -48,6 +48,10 @@ class TDTestCase: tdSql.error("select leastsquares(col7, 1, 1) from test1") tdSql.error("select leastsquares(col8, 1, 1) from test1") tdSql.error("select leastsquares(col9, 1, 1) from test1") + tdSql.error("select leastsquares(col11, 1, 1) from test") + tdSql.error("select leastsquares(col12, 1, 1) from test") + tdSql.error("select leastsquares(col13, 1, 1) from test") + tdSql.error("select leastsquares(col14, 1, 1) from test") tdSql.query("select leastsquares(col1, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') @@ -61,6 +65,18 @@ class TDTestCase: tdSql.query("select leastsquares(col4, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col11, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col12, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col13, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col14, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col5, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') diff --git a/tests/pytest/functions/function_leastsquares_restart.py b/tests/pytest/functions/function_leastsquares_restart.py index 0221dc0aed865882e375f90c44f52b391d1ffee5..8c38d5dd95ccd4537af623253b08fb2253e4d140 100644 --- a/tests/pytest/functions/function_leastsquares_restart.py +++ b/tests/pytest/functions/function_leastsquares_restart.py @@ -54,6 +54,18 @@ class TDTestCase: tdSql.query("select leastsquares(col4, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col11, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col12, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col13, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col14, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col5, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py index 3bd5031276adfd76574a1a43e311a1d6ec81ab16..c322b6af26362949f7e8b45a58e4c9cef3d9de19 100644 --- a/tests/pytest/functions/function_max.py +++ b/tests/pytest/functions/function_max.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,12 +63,33 @@ class TDTestCase: tdSql.query("select max(col4) from test1") tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col11) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col12) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col13) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col14) from test1") + tdSql.checkData(0, 0, np.max(intData)) tdSql.query("select max(col5) from test1") tdSql.checkData(0, 0, np.max(floatData)) tdSql.query("select max(col6) from test1") tdSql.checkData(0, 0, np.max(floatData)) + + # test case: https://jira.taosdata.com:18080/browse/TD-2583 + tdSql.execute("create database test days 2") + tdSql.execute("create table car(ts timestamp, speed int)") + tdSql.execute("insert into car values(now, -1)") + tdSql.execute("insert into car values(now-10d, null)") + + tdSql.query("select max(speed) from car") + tdSql.checkData(0, 0, -1) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_max_restart.py b/tests/pytest/functions/function_max_restart.py index 8d1827561758f1ceebffab877aabd6cd79166d42..a04fcedd8b908b030932d5d182b8cf20c28e2685 100644 --- a/tests/pytest/functions/function_max_restart.py +++ b/tests/pytest/functions/function_max_restart.py @@ -34,7 +34,7 @@ class TDTestCase: floatData = [] #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): intData.append(i + 1) @@ -61,6 +61,18 @@ class TDTestCase: tdSql.query("select max(col4) from test1") tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col11) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col12) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col13) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col14) from test1") + tdSql.checkData(0, 0, np.max(intData)) tdSql.query("select max(col5) from test1") tdSql.checkData(0, 0, np.max(floatData)) diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py index bc180bc22414a4a3aa3d233438b10a04fd08bc2b..b4d6d58f7ce01b97dd7f5f569409a0288fb0389b 100644 --- a/tests/pytest/functions/function_min.py +++ b/tests/pytest/functions/function_min.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,12 +63,33 @@ class TDTestCase: tdSql.query("select min(col4) from test1") tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col11) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col12) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col13) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col14) from test1") + tdSql.checkData(0, 0, np.min(intData)) tdSql.query("select min(col5) from test1") tdSql.checkData(0, 0, np.min(floatData)) tdSql.query("select min(col6) from test1") tdSql.checkData(0, 0, np.min(floatData)) + + # test case: https://jira.taosdata.com:18080/browse/TD-2583 + tdSql.execute("create database test days 2") + tdSql.execute("create table car(ts timestamp, speed int)") + tdSql.execute("insert into car values(now, 1)") + tdSql.execute("insert into car values(now-10d, null)") + + tdSql.query("select min(speed) from car") + tdSql.checkData(0, 0, 1) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_min_restart.py b/tests/pytest/functions/function_min_restart.py index c8329bffa5525c2386d264010acf7231059e344d..fc2cb761cae164c8c04992768133d1e00a16fccb 100644 --- a/tests/pytest/functions/function_min_restart.py +++ b/tests/pytest/functions/function_min_restart.py @@ -58,6 +58,18 @@ class TDTestCase: tdSql.query("select min(col4) from test1") tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col11) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col12) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col13) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col14) from test1") + tdSql.checkData(0, 0, np.min(intData)) tdSql.query("select min(col5) from test1") tdSql.checkData(0, 0, np.min(floatData)) diff --git a/tests/pytest/functions/function_operations.py b/tests/pytest/functions/function_operations.py index 930c23d8a60d84bcc57fa8952e4d8c8e847ec138..e703147b6722f848263f170b89ce9f972f7a2435 100644 --- a/tests/pytest/functions/function_operations.py +++ b/tests/pytest/functions/function_operations.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # min verifacation tdSql.error("select ts + col1 from test") @@ -51,9 +51,9 @@ class TDTestCase: tdSql.checkRows(10) tdSql.checkData(0, 0, 2.0) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") tdSql.checkRows(10) - tdSql.checkData(0, 0, 3.2) + tdSql.checkData(0, 0, 7.2) tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11)) tdSql.query("select col1 + col2 from test1") @@ -64,11 +64,42 @@ class TDTestCase: tdSql.checkRows(11) tdSql.checkData(10, 0, None) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") tdSql.checkRows(11) tdSql.checkData(10, 0, None) + # test for tarithoperator.c coverage + tdSql.execute("insert into test1 values(1537146000010,1,NULL,9,8,1.2,1.3,0,1,1,5,4,3,2)") + tdSql.execute("insert into test1 values(1537146000011,2,1,NULL,9,1.2,1.3,1,2,2,6,5,4,3)") + tdSql.execute("insert into test1 values(1537146000012,3,2,1,NULL,1.2,1.3,0,3,3,7,6,5,4)") + tdSql.execute("insert into test1 values(1537146000013,4,3,2,1,1.2,1.3,1,4,4,8,7,6,5)") + tdSql.execute("insert into test1 values(1537146000014,5,4,3,2,1.2,1.3,0,5,5,9,8,7,6)") + tdSql.execute("insert into test1 values(1537146000015,6,5,4,3,1.2,1.3,1,6,6,NULL,9,8,7)") + tdSql.execute("insert into test1 values(1537146000016,7,6,5,4,1.2,1.3,0,7,7,1,NULL,9,8)") + tdSql.execute("insert into test1 values(1537146000017,8,7,6,5,1.2,1.3,1,8,8,2,1,NULL,9)") + tdSql.execute("insert into test1 values(1537146000018,9,8,7,6,1.2,1.3,0,9,9,3,2,1,NULL)") + tdSql.execute("insert into test1 values(1537146000019,NULL,9,8,7,1.2,1.3,1,10,10,4,3,2,1)") + self.ts = self.ts + self.rowNum + 10 + + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, 1.1, 1, NULL, '涛思数据3', 1, 1, 1, 1)" % ( self.ts + self.rowNum + 1 )) + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, 1.1, 1, 'taosdata', NULL, 1, 1, 1, 1)" % ( self.ts + self.rowNum + 2 )) + tdSql.execute("insert into test1 values(%d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" % ( self.ts + self.rowNum + 3 )) + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, NULL, 1.1, 1, NULL, '涛思数据3', 1, 1, 1, 1)" % ( self.ts + self.rowNum + 4 )) + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, NULL, 1, 'taosdata', NULL, 1, 1, 1, 1)" % ( self.ts + self.rowNum + 5 )) + self.rowNum = self.rowNum + 5 + + col_list = [ 'col1' , 'col2' , 'col3' , 'col4' , 'col5' , 'col6' , 'col7' , 'col8' , 'col9' , 'col11' , 'col12' , 'col13' , 'col14' , '1' , '1.1' , 'NULL' ] + op_list = [ '+' , '-' , '*' , '/' , '%' ] + err_list = [ 'col7' , 'col8' , 'col9' , 'NULL' ] + for i in col_list : + for j in col_list : + for k in op_list : + sql = " select %s %s %s from test1 " % ( i , k , j ) + if i in err_list or j in err_list: + tdSql.error(sql) + else: + tdSql.query(sql) def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/functions/function_operations_restart.py b/tests/pytest/functions/function_operations_restart.py index 6e3990264cb91d8c063fe638a5f77231ef7be424..3ba787a4a0f3f8af74f461f4e668078d726a428d 100644 --- a/tests/pytest/functions/function_operations_restart.py +++ b/tests/pytest/functions/function_operations_restart.py @@ -41,24 +41,24 @@ class TDTestCase: tdSql.error("select col1 + col9 from test1") tdSql.query("select col1 + col2 from test1") - tdSql.checkRows(11) + tdSql.checkRows(25) tdSql.checkData(0, 0, 2.0) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 3.2) + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") + tdSql.checkRows(25) + tdSql.checkData(0, 0, 7.2) #tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11)) tdSql.query("select col1 + col2 from test1") - tdSql.checkRows(11) + tdSql.checkRows(25) tdSql.checkData(10, 0, None) tdSql.query("select col1 + col2 * col3 from test1") - tdSql.checkRows(11) + tdSql.checkRows(25) tdSql.checkData(10, 0, None) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") - tdSql.checkRows(11) + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") + tdSql.checkRows(25) tdSql.checkData(10, 0, None) diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py index e63d65f2e6a429015e2b4d7dcbe5e8c9884eea5e..9b5e209d33d1f8e97c5baf83898dd62746c11dd8 100644 --- a/tests/pytest/functions/function_percentile.py +++ b/tests/pytest/functions/function_percentile.py @@ -34,10 +34,10 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') for i in range(self.rowNum): - tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -103,6 +103,58 @@ class TDTestCase: tdSql.query("select apercentile(col4, 100) from test") print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col11, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col11, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col11, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col12, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col12, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col12, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col12, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col13, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col13, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col13, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col13, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col14, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col14, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col14, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col14, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 0) from test") print("query result: %s" % tdSql.getData(0, 0)) print("array result: %s" % np.percentile(floatData, 0)) @@ -142,6 +194,16 @@ class TDTestCase: tdSql.error("select percentile(voltage, 20) from meters") tdSql.query("select apercentile(voltage, 20) from meters") print("apercentile result: %s" % tdSql.getData(0, 0)) + + # Test case for: https://jira.taosdata.com:18080/browse/TD-2609 + + # modified for : https://jira.taosdata.com:18080/browse/TD-2627 + tdSql.execute("create table st(ts timestamp, k int)") + tdSql.execute("insert into st values(now, -100)(now+1a,-99)") + tdSql.query("select apercentile(k, 20) from st") + tdSql.checkData(0, 0, -100.00) + + def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_percentile_restart.py b/tests/pytest/functions/function_percentile_restart.py index ade10282dedd655bc7ccb912a30a2a3e88a72223..5e6898784b55bbffb5e01946fff9dc60e281ebff 100644 --- a/tests/pytest/functions/function_percentile_restart.py +++ b/tests/pytest/functions/function_percentile_restart.py @@ -99,6 +99,58 @@ class TDTestCase: tdSql.query("select apercentile(col4, 100) from test") print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col11, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col11, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col11, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col12, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col12, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col12, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col12, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col13, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col13, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col13, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col13, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col14, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col14, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col14, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col14, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 0) from test") print("query result: %s" % tdSql.getData(0, 0)) print("array result: %s" % np.percentile(floatData, 0)) diff --git a/tests/pytest/functions/function_spread.py b/tests/pytest/functions/function_spread.py index 462933597388090f069f9319751aaea032ea4133..b40f57b78da0f937e06bd304764c6b660f82488e 100644 --- a/tests/pytest/functions/function_spread.py +++ b/tests/pytest/functions/function_spread.py @@ -31,9 +31,9 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") - tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ', 0, 0, 0, 0)" % (self.ts - 1)) # spread verifacation tdSql.query("select spread(ts) from test1") @@ -55,6 +55,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + tdSql.query("select spread(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query("select spread(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -64,8 +80,8 @@ class TDTestCase: tdSql.checkData(0, 0, 0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.error("select spread(col7) from test") tdSql.error("select spread(col7) from test1") @@ -90,6 +106,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select spread(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select spread(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_spread_restart.py b/tests/pytest/functions/function_spread_restart.py index ba71da0a18ac2e1e2a1ead6ab3de9ff5cae6a600..134d92c9c38c1ae37e9ea1a0a91d0e93957a7e2a 100644 --- a/tests/pytest/functions/function_spread_restart.py +++ b/tests/pytest/functions/function_spread_restart.py @@ -36,7 +36,7 @@ class TDTestCase: tdSql.error("select spread(col8) from test1") tdSql.error("select spread(col9) from test") tdSql.error("select spread(col9) from test1") - + tdSql.query("select spread(col1) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) @@ -53,6 +53,23 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select spread(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_stddev.py b/tests/pytest/functions/function_stddev.py index 23df415aa35feecf84ba40a50604d2b86fc7f008..51e005936183853ee2a4c9cc58bd5dd192b99006 100644 --- a/tests/pytest/functions/function_stddev.py +++ b/tests/pytest/functions/function_stddev.py @@ -34,22 +34,24 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) # stddev verifacation tdSql.error("select stddev(ts) from test1") - tdSql.error("select stddev(col1) from test") - tdSql.error("select stddev(col2) from test") - tdSql.error("select stddev(col3) from test") - tdSql.error("select stddev(col4) from test") - tdSql.error("select stddev(col5) from test") - tdSql.error("select stddev(col6) from test") + + # stddev support super table now + # tdSql.error("select stddev(col1) from test") + # tdSql.error("select stddev(col2) from test") + # tdSql.error("select stddev(col3) from test") + # tdSql.error("select stddev(col4) from test") + # tdSql.error("select stddev(col5) from test") + # tdSql.error("select stddev(col6) from test") tdSql.error("select stddev(col7) from test1") tdSql.error("select stddev(col8) from test1") tdSql.error("select stddev(col9) from test1") @@ -66,6 +68,18 @@ class TDTestCase: tdSql.query("select stddev(col4) from test1") tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col11) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col12) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col13) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col14) from test1") + tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col5) from test1") tdSql.checkData(0, 0, np.std(floatData)) diff --git a/tests/pytest/functions/function_stddev_restart.py b/tests/pytest/functions/function_stddev_restart.py index ec413b94b20a89c210131188d2a09e38cdbc721c..48f9821d89473b66134b5ece06f1d0c417c24b36 100644 --- a/tests/pytest/functions/function_stddev_restart.py +++ b/tests/pytest/functions/function_stddev_restart.py @@ -39,12 +39,6 @@ class TDTestCase: # stddev verifacation tdSql.error("select stddev(ts) from test1") - tdSql.error("select stddev(col1) from test") - tdSql.error("select stddev(col2) from test") - tdSql.error("select stddev(col3) from test") - tdSql.error("select stddev(col4) from test") - tdSql.error("select stddev(col5) from test") - tdSql.error("select stddev(col6) from test") tdSql.error("select stddev(col7) from test1") tdSql.error("select stddev(col8) from test1") tdSql.error("select stddev(col9) from test1") @@ -61,6 +55,18 @@ class TDTestCase: tdSql.query("select stddev(col4) from test1") tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col11) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col12) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col13) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col14) from test1") + tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col5) from test1") tdSql.checkData(0, 0, np.std(floatData)) diff --git a/tests/pytest/functions/function_stddev_td2555.py b/tests/pytest/functions/function_stddev_td2555.py new file mode 100644 index 0000000000000000000000000000000000000000..d7c820c37644397d18cc090afb1ee82efeaecf6c --- /dev/null +++ b/tests/pytest/functions/function_stddev_td2555.py @@ -0,0 +1,105 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 100 + self.ts = 1537146000000 + self.clist1 = [] + self.clist2 = [] + self.clist3 = [] + self.clist4 = [] + self.clist5 = [] + self.clist6 = [] + self.clist11 = [] + self.clist12 = [] + self.clist13 = [] + self.clist14 = [] + + def getData(self): + for i in range(tdSql.queryRows): + for j in range(6): + exec('self.clist{}.append(tdSql.queryResult[i][j+1])'.format(j+1)) + for j in range(11,15): + exec('self.clist{}.append(tdSql.queryResult[i][j-1])'.format(j)) + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(cid int,gbid binary(20),loc nchar(20))''') + tdSql.execute("create table test1 using test tags(1,'beijing','北京')") + tdSql.execute("create table test2 using test tags(2,'shanghai','深圳')") + tdSql.execute("create table test3 using test tags(2,'shenzhen','深圳')") + tdSql.execute("create table test4 using test tags(1,'shanghai','上海')") + for j in range(4): + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(now-%dh, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (j+1,i, i + 1, i + 1, i + 1, i + 1, i + i * 0.1, i * 1.5, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + # stddev verifacation + tdSql.error("select stddev(ts) from test") + tdSql.error("select stddev(col7) from test") + tdSql.error("select stddev(col8) from test") + tdSql.error("select stddev(col9) from test") + + con_list = [ + ' where cid = 1 and ts >=now - 1d and ts =now - 1d and ts =now - 1d and ts =now - 1d and ts =now - 1d and ts %d" % self.ts) - tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col2) from test where ts > %d" % self.ts) - tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col3) from test where ts > %d" % self.ts) - tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col4) from test where ts > %d" % self.ts) - tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col5) from test where ts > %d" % self.ts) - tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col6) from test where ts > %d" % self.ts) - tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col11) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col11) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col12) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col12) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col13) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col13) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col14) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col14) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) def stop(self): diff --git a/tests/pytest/functions/function_twa_restart.py b/tests/pytest/functions/function_twa_restart.py index 2025e3c9ddcdd41aa18db949e5db0d059090e469..a066a5a8f3a4e7f28c577fec0ace725c920d5911 100644 --- a/tests/pytest/functions/function_twa_restart.py +++ b/tests/pytest/functions/function_twa_restart.py @@ -28,12 +28,17 @@ class TDTestCase: self.ts = 1537146000000 def run(self): - tdSql.execute("use db") + tdSql.prepare() intData = [] floatData = [] + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -41,23 +46,25 @@ class TDTestCase: tdSql.error("select twa(ts) from test") tdSql.error("select twa(ts) from test1") - tdSql.error("select twa(col1) from test") - tdSql.error("select twa(col1) from test1") + tdSql.error("select twa(col1) from test") tdSql.error("select twa(col2) from test") - tdSql.error("select twa(col2) from test1") tdSql.error("select twa(col3) from test") - tdSql.error("select twa(col3) from test1") - tdSql.error("select twa(col4) from test") - tdSql.error("select twa(col4) from test1") + tdSql.error("select twa(col4) from test") + + tdSql.error("select twa(col11) from test") + + tdSql.error("select twa(col12) from test") + + tdSql.error("select twa(col13) from test") + + tdSql.error("select twa(col14) from test") tdSql.error("select twa(col5) from test") - tdSql.error("select twa(col5) from test1") - tdSql.error("select twa(col6) from test") - tdSql.error("select twa(col6) from test1") + tdSql.error("select twa(col6) from test") tdSql.error("select twa(col7) from test") tdSql.error("select twa(col7) from test1") @@ -67,59 +74,35 @@ class TDTestCase: tdSql.error("select twa(col9) from test") tdSql.error("select twa(col9) from test1") - - tdSql.error("select twa(col1) from test where ts > %d" % self.ts) - tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col2) from test where ts > %d" % self.ts) - tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col3) from test where ts > %d" % self.ts) - tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col4) from test where ts > %d" % self.ts) - tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col5) from test where ts > %d" % self.ts) - tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col6) from test where ts > %d" % self.ts) - tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col11) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col11) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col12) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col12) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col13) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col13) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col14) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col14) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) def stop(self): @@ -127,4 +110,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/functions/function_twa_test2.py b/tests/pytest/functions/function_twa_test2.py new file mode 100644 index 0000000000000000000000000000000000000000..b20f14357e22d38162b5d69cc3ae643f700f8754 --- /dev/null +++ b/tests/pytest/functions/function_twa_test2.py @@ -0,0 +1,157 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table t1(ts timestamp, c int)") + for i in range(self.rowNum): + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i * 10000, i + 1)) + + # twa verifacation + tdSql.query("select twa(c) from t1 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 5.5) + + tdSql.query("select twa(c) from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 5.5) + + tdSql.query("select twa(c) from t1 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(10s)") + tdSql.checkRows(10) + tdSql.checkData(0, 1, 1.49995) + tdSql.checkData(1, 1, 2.49995) + tdSql.checkData(2, 1, 3.49995) + tdSql.checkData(3, 1, 4.49995) + tdSql.checkData(4, 1, 5.49995) + tdSql.checkData(5, 1, 6.49995) + tdSql.checkData(6, 1, 7.49995) + tdSql.checkData(7, 1, 8.49995) + tdSql.checkData(8, 1, 9.49995) + tdSql.checkData(9, 1, 10) + + tdSql.query("select twa(c) from t1 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(10s) sliding(5s)") + tdSql.checkRows(20) + tdSql.checkData(0, 1, 1.24995) + tdSql.checkData(1, 1, 1.49995) + tdSql.checkData(2, 1, 1.99995) + tdSql.checkData(3, 1, 2.49995) + tdSql.checkData(4, 1, 2.99995) + tdSql.checkData(5, 1, 3.49995) + tdSql.checkData(6, 1, 3.99995) + tdSql.checkData(7, 1, 4.49995) + tdSql.checkData(8, 1, 4.99995) + tdSql.checkData(9, 1, 5.49995) + tdSql.checkData(10, 1, 5.99995) + tdSql.checkData(11, 1, 6.49995) + tdSql.checkData(12, 1, 6.99995) + tdSql.checkData(13, 1, 7.49995) + tdSql.checkData(14, 1, 7.99995) + tdSql.checkData(15, 1, 8.49995) + tdSql.checkData(16, 1, 8.99995) + tdSql.checkData(17, 1, 9.49995) + tdSql.checkData(18, 1, 9.75000) + tdSql.checkData(19, 1, 10) + + tdSql.execute("create table t2(ts timestamp, c int)") + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + 3000)) + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(2s) ") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(2s) sliding(1s) ") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 1) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:04.000' and ts <= '2018-09-17 09:01:30.000' ") + tdSql.checkRows(0) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 08:00:00.000' and ts <= '2018-09-17 09:00:00.000' ") + tdSql.checkRows(0) + + tdSql.execute("create table t3(ts timestamp, c int)") + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts)) + tdSql.execute("insert into t3 values(%d, -2)" % (self.ts + 3000)) + + tdSql.query("select twa(c) from t3 where ts >= '2018-09-17 08:59:00.000' and ts <= '2018-09-17 09:01:30.000'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, -0.5) + + tdSql.query("select twa(c) from t3 where ts >= '2018-09-17 08:59:00.000' and ts <= '2018-09-17 09:01:30.000' interval(1s)") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.5005) + tdSql.checkData(1, 1, -2) + + tdSql.query("select twa(c) from t3 where ts >= '2018-09-17 08:59:00.000' and ts <= '2018-09-17 09:01:30.000' interval(2s) sliding(1s)") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 0.5005) + tdSql.checkData(1, 1, 0.0005) + tdSql.checkData(2, 1, -1.5) + tdSql.checkData(3, 1, -2) + + #TD-2533 twa+interval with large records + tdSql.execute("create table t4(ts timestamp, c int)") + sql = 'insert into t4 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i * 500, i + 1) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t4 values ' + tdSql.execute(sql) + tdSql.query('select twa(c) from t4 interval(10s)') + tdSql.checkData(0,1,10.999) + + # Test case: https://jira.taosdata.com:18080/browse/TD-2624 + tdSql.execute("create database test keep 7300") + tdSql.execute("use test") + tdSql.execute("create table st(ts timestamp, k int)") + tdSql.execute("insert into st values('2011-01-02 18:42:45.326', -1)") + tdSql.execute("insert into st values('2020-07-30 17:44:06.283', 0)") + tdSql.execute("insert into st values('2020-07-30 17:44:19.578', 9999999)") + tdSql.execute("insert into st values('2020-07-30 17:46:06.417', NULL)") + tdSql.execute("insert into st values('2020-11-09 18:42:25.538', 0)") + tdSql.execute("insert into st values('2020-12-29 17:43:11.641', 0)") + tdSql.execute("insert into st values('2020-12-29 18:43:17.129', 0)") + tdSql.execute("insert into st values('2020-12-29 18:46:19.109', NULL)") + tdSql.execute("insert into st values('2021-01-03 18:40:40.065', 0)") + + tdSql.query("select twa(k),first(ts) as taos1 from st where k <50 interval(17s)") + tdSql.checkRows(6) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index ce3d1c0c673e6e62fdd2e6ac83769eb3d7a8ca71..502c859dad85a8fcceee7e0efaaa46ab9bded02a 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -5,13 +5,24 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & -./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4 -pidof taosd|xargs kill +IN_TDINTERNAL="community" +TDIR=`pwd` +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../../.. +fi +TOP_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep -v community|head -n1` +nohup $TAOSD_DIR >/dev/null & +cd - +./crash_gen.sh --valgrind -p -t 10 -s 500 -b 4 +pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` do +memError=(${memError//,/}) if [ -n "$memError" ]; then if [ "$memError" -gt 12 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ @@ -23,12 +34,18 @@ done grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'` do - +defiMemError=(${defiMemError//,/}) if [ -n "$defiMemError" ]; then - if [ "$defiMemError" -gt 3 ]; then - echo -e "${RED} ## Memory errors number valgrind reports \ + if [ "$defiMemError" -gt 0 -a "$defiMemError" -lt 1013 ]; then + cat valgrind.err + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + elif [ "$defiMemError" -gt 1013 ];then #add for azure + cat valgrind.err + echo -e "${RED} ## Memory errors number valgrind reports \ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" - exit 8 + exit 8 fi fi -done \ No newline at end of file +done diff --git a/tests/pytest/import_merge/importCacheFileH.py b/tests/pytest/import_merge/importCacheFileH.py index 3398f7bdad8d7524e223f0a2847b579081686476..33724dfa6889903a34836844271f5843c7cb36e4 100644 --- a/tests/pytest/import_merge/importCacheFileH.py +++ b/tests/pytest/import_merge/importCacheFileH.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHO.py b/tests/pytest/import_merge/importCacheFileHO.py index 19520dc3d048a267c03850b608d47db20394c43d..9f552978f9bb5b0be72eb7cba7ebd073114e740b 100644 --- a/tests/pytest/import_merge/importCacheFileHO.py +++ b/tests/pytest/import_merge/importCacheFileHO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHPO.py b/tests/pytest/import_merge/importCacheFileHPO.py index 9e0a57fb462cdb6ae1dee3c8bfee4488c99a76a0..f2d0a3ddbfb4c0a2a111aab423e32c0ed4683e47 100644 --- a/tests/pytest/import_merge/importCacheFileHPO.py +++ b/tests/pytest/import_merge/importCacheFileHPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileS.py b/tests/pytest/import_merge/importCacheFileS.py index 2f0af569e564b64cbb88c048762e8d41dcf21cbd..250952b2377f9d164c16801b6ead5f695d4f7c43 100644 --- a/tests/pytest/import_merge/importCacheFileS.py +++ b/tests/pytest/import_merge/importCacheFileS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileSub.py b/tests/pytest/import_merge/importCacheFileSub.py index 300bb6e8d09f4484e22e64a0f2caf50cab7c8ab4..663bdcd6351c02a91da309502e0a7f5ec9878086 100644 --- a/tests/pytest/import_merge/importCacheFileSub.py +++ b/tests/pytest/import_merge/importCacheFileSub.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileT.py b/tests/pytest/import_merge/importCacheFileT.py index ab33cf6a93c4bbd7f27967f6a70b61fc62230f65..f2feeeacff11a4c8a2d65feaa08cd1b57264e0c9 100644 --- a/tests/pytest/import_merge/importCacheFileT.py +++ b/tests/pytest/import_merge/importCacheFileT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTO.py b/tests/pytest/import_merge/importCacheFileTO.py index 00e22da976618b4ee870a5702542f669693e3727..421ca6a755425690e7ec76d9a932ce11a65cb141 100644 --- a/tests/pytest/import_merge/importCacheFileTO.py +++ b/tests/pytest/import_merge/importCacheFileTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTPO.py b/tests/pytest/import_merge/importCacheFileTPO.py index c6089e1d687fea6ac900931a8171e45f4923f0cb..ca582d7d5aad8106d7c3d12439001fda84d5004c 100644 --- a/tests/pytest/import_merge/importCacheFileTPO.py +++ b/tests/pytest/import_merge/importCacheFileTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataH2.py b/tests/pytest/import_merge/importDataH2.py index a21f0c47be2eb11a57182d22b56d4447ff2d7ab6..4de567d943a1fe843ec16dd4589054ebcef76488 100644 --- a/tests/pytest/import_merge/importDataH2.py +++ b/tests/pytest/import_merge/importDataH2.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO.py b/tests/pytest/import_merge/importDataHO.py index fdcaedd83c99482914805cb65a067cbcbc78a128..f40de71d6d3967aac6d851deae2810aed63bc3c8 100644 --- a/tests/pytest/import_merge/importDataHO.py +++ b/tests/pytest/import_merge/importDataHO.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO2.py b/tests/pytest/import_merge/importDataHO2.py index b094701132677d6e72e834b16b7edc106682be14..518975e0b727db0d51bd6f5b68e690356988e2fe 100644 --- a/tests/pytest/import_merge/importDataHO2.py +++ b/tests/pytest/import_merge/importDataHO2.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHPO.py b/tests/pytest/import_merge/importDataHPO.py index 9d74c0c3522eba3687618132c27ce95f86f91b91..c7f0625d4bf09de49066c9ea19d4936bdd00e811 100644 --- a/tests/pytest/import_merge/importDataHPO.py +++ b/tests/pytest/import_merge/importDataHPO.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastH.py b/tests/pytest/import_merge/importDataLastH.py index c8e5f62423dc21254d215fac41926aeb9ce5ba40..f736cb925a405c8588d01faad0b08759127962b4 100644 --- a/tests/pytest/import_merge/importDataLastH.py +++ b/tests/pytest/import_merge/importDataLastH.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHO.py b/tests/pytest/import_merge/importDataLastHO.py index 33215997a4220b65228b630ebc350bf9dae15046..b1eebfdd47ed1fbb5185b7547927ea50a29bc64e 100644 --- a/tests/pytest/import_merge/importDataLastHO.py +++ b/tests/pytest/import_merge/importDataLastHO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHPO.py b/tests/pytest/import_merge/importDataLastHPO.py index fa8542f35b8e9639db20dc2e1886fca36140e825..9aa8127f50799039ceaa48dee957cdb903b890f0 100644 --- a/tests/pytest/import_merge/importDataLastHPO.py +++ b/tests/pytest/import_merge/importDataLastHPO.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastS.py b/tests/pytest/import_merge/importDataLastS.py index 2f595fef542955e616ef2e6ebcce7f23bce71d80..2c1559d2903a4fde104603bb2ad5945ea8eddd1e 100644 --- a/tests/pytest/import_merge/importDataLastS.py +++ b/tests/pytest/import_merge/importDataLastS.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastSub.py b/tests/pytest/import_merge/importDataLastSub.py index bfcad2d252cccb2404b3989c474310e0a19afe2e..415d1fcba27921267832323e6c3177678a215c9b 100644 --- a/tests/pytest/import_merge/importDataLastSub.py +++ b/tests/pytest/import_merge/importDataLastSub.py @@ -32,7 +32,7 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.deploy(1) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') @@ -60,9 +60,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 10 data totally repetitive") diff --git a/tests/pytest/import_merge/importDataLastT.py b/tests/pytest/import_merge/importDataLastT.py index 08e944eb6836a6f7f938639b40a6124f7bb64ec6..839320cc22352f1009d0d786aa58cf8827fc8ff6 100644 --- a/tests/pytest/import_merge/importDataLastT.py +++ b/tests/pytest/import_merge/importDataLastT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTO.py b/tests/pytest/import_merge/importDataLastTO.py index a82c0541414298accfdfb447c318ffafd88e5852..0c35519fcf76e7d2faab3a2cb2c4cc702fe93f0b 100644 --- a/tests/pytest/import_merge/importDataLastTO.py +++ b/tests/pytest/import_merge/importDataLastTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTPO.py b/tests/pytest/import_merge/importDataLastTPO.py index ff75a1b2ae87891853fa93f0f49fff339cf6e488..c7864ae14d68a3ef8860125920b3286921f65356 100644 --- a/tests/pytest/import_merge/importDataLastTPO.py +++ b/tests/pytest/import_merge/importDataLastTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataS.py b/tests/pytest/import_merge/importDataS.py index 37627e8d6bcc8319805275525ef7874132e0da4d..f0d56a5016fe5b059e1b51719b799d801219c2ca 100644 --- a/tests/pytest/import_merge/importDataS.py +++ b/tests/pytest/import_merge/importDataS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataSub.py b/tests/pytest/import_merge/importDataSub.py index 17e2b141b7014fd2ce4d31f8c335e122aca730ba..1643cc89155507be6c5fa2c072755a8439b21aab 100644 --- a/tests/pytest/import_merge/importDataSub.py +++ b/tests/pytest/import_merge/importDataSub.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataT.py b/tests/pytest/import_merge/importDataT.py index b0b7b82b790e81c475c714ba570a9a6f1801cc13..c23ad9a60533d94d34d11c70ea2743a1b8ab5c01 100644 --- a/tests/pytest/import_merge/importDataT.py +++ b/tests/pytest/import_merge/importDataT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTO.py b/tests/pytest/import_merge/importDataTO.py index c0b57136af7a51c1d184eb5789d95efc0148118c..b4c0d1a8adb6d4a3751d5a4a6547d0d03eab793b 100644 --- a/tests/pytest/import_merge/importDataTO.py +++ b/tests/pytest/import_merge/importDataTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTPO.py b/tests/pytest/import_merge/importDataTPO.py index 8a1c9264b433484e768cdfffbccdb0966e3d15cd..38e8cfe9096135f1f9301d89bc8c6269d045e78f 100644 --- a/tests/pytest/import_merge/importDataTPO.py +++ b/tests/pytest/import_merge/importDataTPO.py @@ -57,9 +57,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 20 data later with partly overlap") diff --git a/tests/pytest/import_merge/importHORestart.py b/tests/pytest/import_merge/importHORestart.py index f74c4c76d616753f8cadc87ba715e473eae0fb46..05b5b42afff14c695690b7651bd385ee78f3bafb 100644 --- a/tests/pytest/import_merge/importHORestart.py +++ b/tests/pytest/import_merge/importHORestart.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHPORestart.py b/tests/pytest/import_merge/importHPORestart.py index e5f79fbe6c2eb686d63f5b7de81651a98072a986..f9167709130d67b2909d3e9efd1acc482638836d 100644 --- a/tests/pytest/import_merge/importHPORestart.py +++ b/tests/pytest/import_merge/importHPORestart.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHRestart.py b/tests/pytest/import_merge/importHRestart.py index be67039789dd660ddb0be8d56753737010e47e79..c1d50378b20a11b4d562b408d7bec07364969d19 100644 --- a/tests/pytest/import_merge/importHRestart.py +++ b/tests/pytest/import_merge/importHRestart.py @@ -54,7 +54,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importInsertThenImport.py b/tests/pytest/import_merge/importInsertThenImport.py index 292fae8c47b398dcbacdb8b76592826bcc8b6019..1372177de36c1f67ab85ba4a63c26553ee4a7de4 100644 --- a/tests/pytest/import_merge/importInsertThenImport.py +++ b/tests/pytest/import_merge/importInsertThenImport.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.stop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdLog.info("import 100 sequential data again") diff --git a/tests/pytest/import_merge/importLastH.py b/tests/pytest/import_merge/importLastH.py index 17fa233e3723e1d9e6cc4ec97c8eac2716751f00..2887c71e040b7ea89e42da69e597bcb843a2df5c 100644 --- a/tests/pytest/import_merge/importLastH.py +++ b/tests/pytest/import_merge/importLastH.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHO.py b/tests/pytest/import_merge/importLastHO.py index adb44fc0eab9a4923b0896a1357b24603f1ff3fd..1681212b6b8c5d002e6f3d2b5853534382bb4d10 100644 --- a/tests/pytest/import_merge/importLastHO.py +++ b/tests/pytest/import_merge/importLastHO.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHPO.py b/tests/pytest/import_merge/importLastHPO.py index d8ed2d9ef10dbaa52e92ba50625be25c495091ea..389b3c11ed04b0359564a41c7b383e37e9bcabdd 100644 --- a/tests/pytest/import_merge/importLastHPO.py +++ b/tests/pytest/import_merge/importLastHPO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastS.py b/tests/pytest/import_merge/importLastS.py index bf222a0d5f3c27c5c7774b384799a6a0aa5e4114..e6393bbfae2f41efd1ff740f4e542715f23f8adf 100644 --- a/tests/pytest/import_merge/importLastS.py +++ b/tests/pytest/import_merge/importLastS.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastSub.py b/tests/pytest/import_merge/importLastSub.py index 5a6b9f41502970149257b1aef5c7a37824401768..4ff6cf27bdf582040360e40ce13d28df7dfc8949 100644 --- a/tests/pytest/import_merge/importLastSub.py +++ b/tests/pytest/import_merge/importLastSub.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastT.py b/tests/pytest/import_merge/importLastT.py index 2b1be1fe2b37aa5e4d59bb6ecef7d0bb2180cf9d..145b1bd69097804728020ea1b3b8a0ccf7e86d27 100644 --- a/tests/pytest/import_merge/importLastT.py +++ b/tests/pytest/import_merge/importLastT.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTO.py b/tests/pytest/import_merge/importLastTO.py index ce189f63718956991982bd33296eb71bd5a82d45..a159dac4acea9c1c278025e157ff3fe09487e858 100644 --- a/tests/pytest/import_merge/importLastTO.py +++ b/tests/pytest/import_merge/importLastTO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTPO.py b/tests/pytest/import_merge/importLastTPO.py index 627d090855ac320f56a90fb647101e8ccc315acd..453b922a68bc7a051aa60d51943ffb759444ee5b 100644 --- a/tests/pytest/import_merge/importLastTPO.py +++ b/tests/pytest/import_merge/importLastTPO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importSRestart.py b/tests/pytest/import_merge/importSRestart.py index 29f5a19b54fb0cbf1eb81a66c66813f3fcdd9982..58f0f60e0af0660e4550966c2f0b1741aefbc33c 100644 --- a/tests/pytest/import_merge/importSRestart.py +++ b/tests/pytest/import_merge/importSRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importSubRestart.py b/tests/pytest/import_merge/importSubRestart.py index b1a6f30c436c5aa52d5ee4affa03d18c5475ced7..85594e1772409f7c24a5d2ce8aa25b6ba7567a5d 100644 --- a/tests/pytest/import_merge/importSubRestart.py +++ b/tests/pytest/import_merge/importSubRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTORestart.py b/tests/pytest/import_merge/importTORestart.py index 07eb6c28cb79c16504f2acab036a3b66c3bccabf..10ac77d759192de329ab0818b79610259d05b7d7 100644 --- a/tests/pytest/import_merge/importTORestart.py +++ b/tests/pytest/import_merge/importTORestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTPORestart.py b/tests/pytest/import_merge/importTPORestart.py index 10bbf3efced90b6c255326c0e8b8e0c106a3a038..ab86a0247f179bd6dab397a069ca9b4aaf4da61f 100644 --- a/tests/pytest/import_merge/importTPORestart.py +++ b/tests/pytest/import_merge/importTPORestart.py @@ -68,7 +68,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTRestart.py b/tests/pytest/import_merge/importTRestart.py index 63a9368eca2b6a3377b32dc6a3948cf5a0ba1186..6106bdd75310369d53ce8148f937655301b64c12 100644 --- a/tests/pytest/import_merge/importTRestart.py +++ b/tests/pytest/import_merge/importTRestart.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/insert/basic_unsigned.py b/tests/pytest/insert/basic_unsigned.py new file mode 100644 index 0000000000000000000000000000000000000000..ff7e0e5e4ad57a35d574d88ea179f713c8a8b611 --- /dev/null +++ b/tests/pytest/insert/basic_unsigned.py @@ -0,0 +1,56 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + ret = tdSql.execute( + 'create table tb (ts timestamp, speed tinyint unsigned)') + + insertRows = 10 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into tb values (now + %dm, %d)' % + (i, i)) + + tdLog.info("insert earlier data") + tdSql.execute('insert into tb values (now - 5m , NULL)') + tdSql.execute('insert into tb values (now - 6m , 10)') + tdSql.execute('insert into tb values (now - 7m , NULL)') + tdSql.execute('insert into tb values (now - 8m , 254)') + + tdSql.error('insert into tb values (now - 9m, -1)') + tdSql.error('insert into tb values (now - 9m, 255)') + + tdSql.query("select * from tb") + tdSql.checkRows(insertRows + 4) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/before_1970.py b/tests/pytest/insert/before_1970.py index cb17b657aad1a3bfdb915c9661bad291b75d6f04..b2c4dc57c7a37b4b119308335d326521fe986f08 100644 --- a/tests/pytest/insert/before_1970.py +++ b/tests/pytest/insert/before_1970.py @@ -41,7 +41,7 @@ class TDTestCase: #TODO : should add more testcases tdSql.execute("insert into test values('1930-12-12 01:19:20.345', 1);") tdSql.execute("insert into test values('1969-12-30 23:59:59.999', 2);") - tdSql.execute("insert into test values(-3600, 3);") + tdSql.execute("insert into test values(-3600001, 3);") tdSql.execute("insert into test values('2020-10-20 14:02:53.770', 4);") print("==============insert data") @@ -61,11 +61,11 @@ class TDTestCase: print("==============step4") tdSql.execute("use demo;") tdSql.query("select * from test;") - # print(tdSql.queryResult) + print(tdSql.queryResult) tdSql.checkRows(4) tdSql.checkData(0,0,'1930-12-12 01:19:20.345000') tdSql.checkData(1,0,'1969-12-30 23:59:59.999000') - tdSql.checkData(2,0,'1970-01-01 07:00:00.000000') + tdSql.checkData(2,0,'1970-01-01 06:59:59.999000') tdSql.checkData(3,0,'2020-10-20 14:02:53.770000') print("==============check data") diff --git a/tests/pytest/insert/boundary2.py b/tests/pytest/insert/boundary2.py new file mode 100644 index 0000000000000000000000000000000000000000..99784b7cd9cfa7fc8d32720442cefa7946f6511f --- /dev/null +++ b/tests/pytest/insert/boundary2.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +import time +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def get_random_string(self, length): + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(length)) + return result_str + + def run(self): + tdSql.prepare() + + startTime = time.time() + print("==============step1") + sql = "create table stb(ts timestamp, " + for i in range(1022): + sql += "col%d binary(14), " % (i + 1) + sql += "col1023 binary(22))" + tdSql.execute(sql) + + for i in range(4096): + sql = "insert into stb values(%d, " + for j in range(1022): + str = "'%s', " % self.get_random_string(14) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + + tdSql.query("select * from stb") + tdSql.checkRows(4096) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select * from stb") + tdSql.checkRows(4096) + + endTime = time.time() + + print("total time %ds" % (endTime - startTime)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py new file mode 100644 index 0000000000000000000000000000000000000000..84fd1d7ccab85d7e4469c779f10f047160ee0565 --- /dev/null +++ b/tests/pytest/insert/insertFromCSVPerformance.py @@ -0,0 +1,131 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +import datetime +import csv +import random +import pandas as pd +import argparse +import os.path + +class insertFromCSVPerformace: + def __init__(self, commitID, dbName, stbName, branchName): + self.commitID = commitID + self.dbName = dbName + self.stbName = stbName + self.branchName = branchName + self.ts = 1500074556514 + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taosperf" + self.conn = taos.connect( + self.host, + self.user, + self.password, + self.config) + + def writeCSV(self): + with open('test3.csv','w', encoding='utf-8', newline='') as csvFile: + writer = csv.writer(csvFile, dialect='excel') + for i in range(1000000): + newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000) + d = datetime.datetime.fromtimestamp(newTimestamp / 1000) + dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) + writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) + + def removCSVHeader(self): + data = pd.read_csv("ordered.csv") + data = data.drop([0]) + data.to_csv("ordered.csv", header = False, index = False) + + def createTables(self): + cursor = self.conn.cursor() + + cursor.execute("create database if not exists %s" % self.dbName) + cursor.execute("use %s" % self.dbName) + cursor.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50)) tags(branch binary(50))" % self.stbName) + cursor.execute("create table if not exists %s using %s tags('%s')" % (self.branchName, self.stbName, self.branchName)) + + cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + + cursor.close() + + def run(self): + cursor = self.conn.cursor() + cursor.execute("use %s" % self.dbName) + print("==================== CSV insert performance ====================") + + totalTime = 0 + for i in range(10): + cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + startTime = time.time() + cursor.execute("insert into t1 file 'outoforder.csv'") + totalTime += time.time() - startTime + cursor.execute("drop table if exists t1") + out_of_order_time = (float) (totalTime / 10) + print("Out of Order - Insert time: %f" % out_of_order_time) + + totalTime = 0 + for i in range(10): + cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + startTime = time.time() + cursor.execute("insert into t2 file 'ordered.csv'") + totalTime += time.time() - startTime + cursor.execute("drop table if exists t2") + + in_order_time = (float) (totalTime / 10) + print("In order - Insert time: %f" % in_order_time) + cursor.execute("insert into %s values(now, %f, %f, '%s')" % (self.branchName, in_order_time, out_of_order_time, self.commitID)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-c', + '--commit-id', + action='store', + default='null', + type=str, + help='git commit id (default: null)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='perf', + type=str, + help='Database name to be created (default: perf)') + parser.add_argument( + '-t', + '--stable-name', + action='store', + default='csv_insert', + type=str, + help='Database name to be created (default: csv_insert)') + parser.add_argument( + '-b', + '--branch-name', + action='store', + default='develop', + type=str, + help='branch name (default: develop)') + + args = parser.parse_args() + perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.stable_name, args.branch_name) + + perftest.createTables() + perftest.run() \ No newline at end of file diff --git a/tests/pytest/insert/metadataUpdate.py b/tests/pytest/insert/metadataUpdate.py new file mode 100644 index 0000000000000000000000000000000000000000..76b9a3ae8f818e3bf3784eb99974ce56b1071af5 --- /dev/null +++ b/tests/pytest/insert/metadataUpdate.py @@ -0,0 +1,67 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from multiprocessing import Process + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def updateMetadata(self): + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = tdDnodes.getSimCfgPath() + + self.conn = taos.connect(host = self.host, user = self.user, password = self.password, config = self.config) + self.cursor = self.conn.cursor() + self.cursor.execute("alter table db.tb add column col2 int") + print("alter table done") + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table if not exists tb (ts timestamp, col1 int)") + tdSql.execute("insert into tb values(%d, 1)" % self.ts) + + print("==============step2") + tdSql.query("select * from tb") + tdSql.checkRows(1) + + p = Process(target=self.updateMetadata, args=()) + p.start() + p.join() + p.terminate() + + tdSql.execute("insert into tb(ts, col1, col2) values(%d, 1, 2)" % (self.ts + 2)) + + print("==============step2") + tdSql.query("select * from tb") + tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index da797f788fcc3cc1ae06843b70da9aaf7230e97c..6489fd31ffd6f595d15d996b92f5c63fc7e2b8a9 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -38,7 +38,7 @@ class RestfulInsert: for i in range(loop): tableID = threadID * tablesPerThread if tableID + i >= self.numOfTables : break - name = 'beijing' if tableID % 2 == 0 else 'shanghai' + name = 'beijing' if (tableID + i) % 2 == 0 else 'shanghai' data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) response = requests.post(self.url, data, headers = self.header) if response.status_code != 200: diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index c69060b5aec16b8d58aa2742e24c44d1ebe543ab..e0446113d6d1fb197490a09ebd1ebe4b5b12e66f 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -44,7 +44,8 @@ class TDTestRetetion: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, self.queryRows, expectRows) os.system("sudo timedatectl set-ntp true") - time.sleep(40) + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) + time.sleep(5) tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) def run(self): @@ -63,7 +64,7 @@ class TDTestRetetion: tdLog.info("=============== step2") tdDnodes.stop(1) os.system("sudo timedatectl set-ntp false") - os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48))) tdDnodes.start(1) cmd = 'insert into test values(now,5);' tdDnodes.stop(1) @@ -79,7 +80,7 @@ class TDTestRetetion: self.checkRows(5,cmd) tdLog.info("=============== step3") tdDnodes.stop(1) - os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48))) tdDnodes.start(1) tdLog.info(cmd) tdSql.execute(cmd) @@ -99,18 +100,19 @@ class TDTestRetetion: tdLog.info(cmd) tdSql.execute(cmd) self.queryRows=tdSql.query('select * from test') - self.checkRows(7,cmd) + self.checkRows(5,cmd) tdLog.info("=============== step5") tdDnodes.stop(1) tdDnodes.start(1) cmd='select * from test where ts > now-1d' self.queryRows=tdSql.query('select * from test where ts > now-1d') - self.checkRows(1,cmd) + self.checkRows(2,cmd) def stop(self): os.system("sudo timedatectl set-ntp true") - time.sleep(40) + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) + time.sleep(5) tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/insert/unsignedBigint.py b/tests/pytest/insert/unsignedBigint.py new file mode 100644 index 0000000000000000000000000000000000000000..b222f2cd0195e14fddf1f10662447cef97f0e841 --- /dev/null +++ b/tests/pytest/insert/unsignedBigint.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bigint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed bigint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 18446744073709551614)") + tdSql.execute("insert into tb values (now+2m, 18446744073709551614)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 18446744073709551614)') + tdSql.checkData(0, 1, 18446744073709551614) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 18446744073709551615) -x step4") + tdSql.error("insert into tb values (now+3m, 18446744073709551615)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now-4m, -1)") + tdSql.error("insert into tb values (now-4m, -1)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/unsignedInt.py b/tests/pytest/insert/unsignedInt.py new file mode 100644 index 0000000000000000000000000000000000000000..ed18999bc415022da34788a79e9045a5f67cf8ee --- /dev/null +++ b/tests/pytest/insert/unsignedInt.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed int unsigned)') + tdSql.execute('create table tb (ts timestamp, speed int unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 4294967294)") + tdSql.execute("insert into tb values (now+2m, 4294967294)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 4294967294)') + tdSql.checkData(0, 1, 4294967294) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 4294967295) -x step4") + tdSql.error("insert into tb values (now+3m, 4294967295)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now-4m, -1)") + tdSql.error("insert into tb values (now-4m, -1)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/unsignedSmallint.py b/tests/pytest/insert/unsignedSmallint.py new file mode 100644 index 0000000000000000000000000000000000000000..9893c470ce1a53a05138dd9d5c18e2b0a9f21374 --- /dev/null +++ b/tests/pytest/insert/unsignedSmallint.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed smallint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed smallint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 65534)") + tdSql.execute("insert into tb values (now+2m, 65534)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 65534)') + tdSql.checkData(0, 1, 65534) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 65535) -x step4") + tdSql.error("insert into tb values (now+3m, 65535)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now-4m, -1)") + tdSql.error("insert into tb values (now-4m, -1)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/unsignedTinyint.py b/tests/pytest/insert/unsignedTinyint.py new file mode 100644 index 0000000000000000000000000000000000000000..5bdfe7580b198ccaffcab7005e88aa72e1c437af --- /dev/null +++ b/tests/pytest/insert/unsignedTinyint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed tinyint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed tinyint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 254)") + tdSql.execute("insert into tb values (now+2m, 254)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 254)') + tdSql.checkData(0, 1, 254) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 255) -x step4") + tdSql.error("insert into tb values (now+3m, 255)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/basic.py b/tests/pytest/multilevel/basic.py new file mode 100644 index 0000000000000000000000000000000000000000..95549409c9f1a1b287fc872260d33f956ccf3510 --- /dev/null +++ b/tests/pytest/multilevel/basic.py @@ -0,0 +1,51 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + cfg={ + '/mnt/data1' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.haveFile('/mnt/data1',1) + tdSql.haveFile('/mnt/data2',1) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/dirNotExist.py b/tests/pytest/multilevel/dirNotExist.py new file mode 100644 index 0000000000000000000000000000000000000000..e895ac31ad1d6361cddfe867e39cea370412cb58 --- /dev/null +++ b/tests/pytest/multilevel/dirNotExist.py @@ -0,0 +1,50 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + cfg={ + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + os.system('rm -rf /mnt/data2') + + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/dirPermissionDenied.py b/tests/pytest/multilevel/dirPermissionDenied.py new file mode 100644 index 0000000000000000000000000000000000000000..db2cdcf757121c501e33f605aceebdf875d5f944 --- /dev/null +++ b/tests/pytest/multilevel/dirPermissionDenied.py @@ -0,0 +1,51 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + cfg={ + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + os.system('chmod 111 /mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/fileDistributionSameLevel.py b/tests/pytest/multilevel/fileDistributionSameLevel.py new file mode 100644 index 0000000000000000000000000000000000000000..83ce85671771c4d2627541e80262ef8c764e45c3 --- /dev/null +++ b/tests/pytest/multilevel/fileDistributionSameLevel.py @@ -0,0 +1,108 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + +dataDir = ['data00','data01','data02','data03','data04'] +dataDict = {'data00':0,'data01':0,'data02':0,'data03':0,'data04':0} +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getfiles(self,ospath): + try: + files = os.listdir(ospath) + for f in files: + path = os.path.join(ospath, f) + if os.path.isfile(path): + if path.endswith('.data'): + for i in dataDir: + if i in path: + dataDict[i] = dataDict[i] + 1 + print(path) + if os.path.isdir(path): + self.getfiles(path) + except Exception as e : + print(str(e)) + + def run(self): + self.ntables = 1000 + self.ts = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + tdLog.info("================= step1") + cfg={ + '10' : 'maxVgroupsPerDb', + '100' : 'maxTablesPerVnode', + '/mnt/data00 0 1' : 'dataDir', + '/mnt/data01 0 0' : 'dataDir', + '/mnt/data02 0 0' : 'dataDir', + '/mnt/data03 0 0' : 'dataDir', + '/mnt/data04 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data00') + tdSql.createDir('/mnt/data01') + tdSql.createDir('/mnt/data02') + tdSql.createDir('/mnt/data03') + tdSql.createDir('/mnt/data04') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.execute("create database test days 1") + tdSql.execute("use test") + + tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") + + for i in range(self.ntables): + tdSql.execute("create table tb%d using stb tags(%d)" %(i, i)) + tdSql.execute("insert into tb%d values(%d, 1)" % (i,self.ts + int (i / 100) * 86400000)) + + tdLog.info("================= step2") + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select * from test.stb") + tdSql.checkRows(1000) + + tdLog.info("================= step3") + tdSql.execute('drop database test') + for i in range(50): + tdSql.execute("create database test%d days 1" %(i)) + tdSql.execute("use test%d" %(i)) + tdSql.execute("create table tb (ts timestamp,i int)") + for j in range(10): + tdSql.execute("insert into tb values(%d, 1)" % (self.ts + int (i / 100) * 86400000)) + tdDnodes.stop(1) + tdDnodes.start(1) + flag = True + for i in range(4): + if dataDict[dataDir[i]] == dataDict[dataDir[i+1]]: + flag = flag & True + else: + flag = flag & False + break + if not flag : tdLog.exit("%s failed, expect not occured" % (sys.argv[0])) + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/illegelLevelMountPoint.py b/tests/pytest/multilevel/illegelLevelMountPoint.py new file mode 100644 index 0000000000000000000000000000000000000000..cbf7fc1fe908fa528713576921555380d4a68ed8 --- /dev/null +++ b/tests/pytest/multilevel/illegelLevelMountPoint.py @@ -0,0 +1,59 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 3 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 -1 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/middleLevelMissing.py b/tests/pytest/multilevel/middleLevelMissing.py new file mode 100644 index 0000000000000000000000000000000000000000..727c2380ae0f3a148a475ae8ebdd01e8ff1f543f --- /dev/null +++ b/tests/pytest/multilevel/middleLevelMissing.py @@ -0,0 +1,64 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 1 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + + tdDnodes.stop(1) + # Test2 2 dataDir + cfg={ + '/mnt/data1 0 1' : 'dataDir', + '/mnt/data1 2 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/moreThan16disks.py b/tests/pytest/multilevel/moreThan16disks.py new file mode 100644 index 0000000000000000000000000000000000000000..407e5aa6970781ce5b511faa1f4f74e7a18e75a8 --- /dev/null +++ b/tests/pytest/multilevel/moreThan16disks.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + cfg={} + for i in range(17): + if i == 0 : + datadir = '/mnt/data%d 0 1' % (i+1) + else: + datadir = '/mnt/data%d 0 0' % (i+1) + cfg.update({ datadir : 'dataDir' }) + tdSql.createDir('/mnt/data%d' % (i+1)) + + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/multiMainMountPoint.py b/tests/pytest/multilevel/multiMainMountPoint.py new file mode 100644 index 0000000000000000000000000000000000000000..c5af816205cdbd325c11d600c560104ca1801b93 --- /dev/null +++ b/tests/pytest/multilevel/multiMainMountPoint.py @@ -0,0 +1,49 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + cfg={ + '/mnt/data1' : 'dataDir', + '/mnt/data2' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/noneMainMountPoint.py b/tests/pytest/multilevel/noneMainMountPoint.py new file mode 100644 index 0000000000000000000000000000000000000000..67fab7c7a24510244a663dbc20f2c212012f712d --- /dev/null +++ b/tests/pytest/multilevel/noneMainMountPoint.py @@ -0,0 +1,49 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + cfg={ + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/retentionTest.py b/tests/pytest/multilevel/retentionTest.py new file mode 100644 index 0000000000000000000000000000000000000000..9ea59d9d5918d851f69acc69704926d97f3f5375 --- /dev/null +++ b/tests/pytest/multilevel/retentionTest.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data00 0 1' : 'dataDir', + '/mnt/data01 0 0' : 'dataDir', + '/mnt/data02 0 0' : 'dataDir', + '/mnt/data10 1 0' : 'dataDir', + '/mnt/data11 1 0' : 'dataDir', + '/mnt/data12 1 0' : 'dataDir', + '/mnt/data20 2 0' : 'dataDir', + '/mnt/data21 2 0' : 'dataDir', + '/mnt/data22 2 0' : 'dataDir' + } + tdSql.createDir('/mnt/data00') + tdSql.createDir('/mnt/data01') + tdSql.createDir('/mnt/data02') + tdSql.createDir('/mnt/data10') + tdSql.createDir('/mnt/data11') + tdSql.createDir('/mnt/data12') + tdSql.createDir('/mnt/data20') + tdSql.createDir('/mnt/data21') + tdSql.createDir('/mnt/data22') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.execute("create database test days 1 keep 15,5,10") + tdSql.execute("use test") + + tdSql.execute("create table tb(ts timestamp, c int)") + + count = 0 + os.system("sudo timedatectl set-ntp false") + + for i in range(self.rowsPerTable): + tdSql.execute("insert into tb values(now, 1)") + count += 1 + tdSql.query("select * from tb") + tdSql.checkRows(count) + tdDnodes.stop(1) + os.system("sudo date -s $(date -d \"${DATE} 1 day\" \"+%Y%m%d\")") + tdDnodes.start(1) + + def stop(self): + os.system("sudo timedatectl set-ntp true") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/threeLevelMountPoint.py b/tests/pytest/multilevel/threeLevelMountPoint.py new file mode 100644 index 0000000000000000000000000000000000000000..d97602eb76889ab265f6bc673f5c7558be5d7fe3 --- /dev/null +++ b/tests/pytest/multilevel/threeLevelMountPoint.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data00 0 1' : 'dataDir', + '/mnt/data01 0 0' : 'dataDir', + '/mnt/data02 0 0' : 'dataDir', + '/mnt/data10 1 0' : 'dataDir', + '/mnt/data11 1 0' : 'dataDir', + '/mnt/data12 1 0' : 'dataDir', + '/mnt/data20 2 0' : 'dataDir', + '/mnt/data21 2 0' : 'dataDir', + '/mnt/data22 2 0' : 'dataDir' + } + tdSql.createDir('/mnt/data00') + tdSql.createDir('/mnt/data01') + tdSql.createDir('/mnt/data02') + tdSql.createDir('/mnt/data10') + tdSql.createDir('/mnt/data11') + tdSql.createDir('/mnt/data12') + tdSql.createDir('/mnt/data20') + tdSql.createDir('/mnt/data21') + tdSql.createDir('/mnt/data22') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(1) + tdSql.haveFile('/mnt/data00',1) + tdSql.haveFile('/mnt/data01',1) + tdSql.haveFile('/mnt/data02',1) + tdSql.haveFile('/mnt/data10',1) + tdSql.haveFile('/mnt/data11',1) + tdSql.haveFile('/mnt/data12',1) + tdSql.haveFile('/mnt/data20',1) + tdSql.haveFile('/mnt/data21',1) + tdSql.haveFile('/mnt/data22',1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/wrongLevelMountPoint.py b/tests/pytest/multilevel/wrongLevelMountPoint.py new file mode 100644 index 0000000000000000000000000000000000000000..00b9eb89c69a21a9d02f4927d508b8275e07a7f7 --- /dev/null +++ b/tests/pytest/multilevel/wrongLevelMountPoint.py @@ -0,0 +1,49 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 1 1' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index 52f5a30f4e83089bc2e6f230c099e4304a94efd4..e6638cbb171ab3d8dc38c13f5767ff2f09c522bb 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -19,13 +19,22 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py +python3 ./test.py -f insert/before_1970.py +python3 bug2265.py +#table python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py python3 ./test.py -f table/create_sensitive.py #python3 ./test.py -f table/tablename-boundary.py +python3 ./test.py -f table/max_table_length.py +python3 ./test.py -f table/alter_column.py +python3 ./test.py -f table/boundary.py +python3 ./test.py -f table/create.py +python3 ./test.py -f table/del_stable.py + # tag python3 ./test.py -f tag_lite/filter.py @@ -126,93 +135,4 @@ python3 ./test.py -f import_merge/importTORestart.py python3 ./test.py -f import_merge/importTPORestart.py python3 ./test.py -f import_merge/importTRestart.py python3 ./test.py -f import_merge/importInsertThenImport.py -python3 ./test.py -f import_merge/importCSV.py -# user -python3 ./test.py -f user/user_create.py -python3 ./test.py -f user/pass_len.py - -# stable -python3 ./test.py -f stable/query_after_reset.py - -# table -python3 ./test.py -f table/del_stable.py - -#query -python3 ./test.py -f query/filter.py -python3 ./test.py -f query/filterCombo.py -python3 ./test.py -f query/queryNormal.py -python3 ./test.py -f query/queryError.py -python3 ./test.py -f query/filterAllIntTypes.py -python3 ./test.py -f query/filterFloatAndDouble.py -python3 ./test.py -f query/filterOtherTypes.py -python3 ./test.py -f query/querySort.py -python3 ./test.py -f query/queryJoin.py -python3 ./test.py -f query/select_last_crash.py -python3 ./test.py -f query/queryNullValueTest.py -python3 ./test.py -f query/queryInsertValue.py -python3 ./test.py -f query/queryConnection.py -python3 ./test.py -f query/queryCountCSVData.py -python3 ./test.py -f query/natualInterval.py -python3 ./test.py -f query/bug1471.py -#python3 ./test.py -f query/dataLossTest.py -python3 ./test.py -f query/bug1874.py -python3 ./test.py -f query/bug1875.py -python3 ./test.py -f query/bug1876.py -python3 ./test.py -f query/bug2218.py - -#stream -python3 ./test.py -f stream/metric_1.py -python3 ./test.py -f stream/new.py -python3 ./test.py -f stream/stream1.py -python3 ./test.py -f stream/stream2.py -#python3 ./test.py -f stream/parser.py -python3 ./test.py -f stream/history.py - -#alter table -python3 ./test.py -f alter/alter_table_crash.py - -# client -python3 ./test.py -f client/client.py -python3 ./test.py -f client/version.py -python3 ./test.py -f client/alterDatabase.py - -# Misc -python3 testCompress.py -python3 testNoCompress.py -python3 testMinTablesPerVnode.py - -# functions -python3 ./test.py -f functions/function_avg.py -r 1 -python3 ./test.py -f functions/function_bottom.py -r 1 -python3 ./test.py -f functions/function_count.py -r 1 -python3 ./test.py -f functions/function_diff.py -r 1 -python3 ./test.py -f functions/function_first.py -r 1 -python3 ./test.py -f functions/function_last.py -r 1 -python3 ./test.py -f functions/function_last_row.py -r 1 -python3 ./test.py -f functions/function_leastsquares.py -r 1 -python3 ./test.py -f functions/function_max.py -r 1 -python3 ./test.py -f functions/function_min.py -r 1 -python3 ./test.py -f functions/function_operations.py -r 1 -python3 ./test.py -f functions/function_percentile.py -r 1 -python3 ./test.py -f functions/function_spread.py -r 1 -python3 ./test.py -f functions/function_stddev.py -r 1 -python3 ./test.py -f functions/function_sum.py -r 1 -python3 ./test.py -f functions/function_top.py -r 1 -#python3 ./test.py -f functions/function_twa.py -r 1 -python3 queryCount.py -python3 ./test.py -f query/queryGroupbyWithInterval.py -python3 client/twoClients.py -python3 test.py -f query/queryInterval.py -python3 test.py -f query/queryFillTest.py - -# tools -python3 test.py -f tools/taosdemoTest.py -python3 test.py -f tools/taosdumpTest.py -python3 test.py -f tools/lowaTest.py - -# subscribe -python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py -python3 test.py -f subscribe/supertable.py - - +python3 ./test.py -f import_merge/importCSV.py \ No newline at end of file diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh index fededea3bb34481aa34c37d4a802ba183494e491..d152ed85fb9e138a6b9d62574bcc8a5119973448 100755 --- a/tests/pytest/pytest_2.sh +++ b/tests/pytest/pytest_2.sh @@ -7,11 +7,10 @@ python3 ./test.py -f update/append_commit_data.py python3 ./test.py -f update/append_commit_last-0.py python3 ./test.py -f update/append_commit_last.py python3 ./test.py -f update/merge_commit_data.py -python3 ./test.py -f update/merge_commit_data-0.py + python3 ./test.py -f update/merge_commit_data2.py python3 ./test.py -f update/merge_commit_data2_update0.py python3 ./test.py -f update/merge_commit_last-0.py python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py -# wal -python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file diff --git a/tests/pytest/pytest_3.sh b/tests/pytest/pytest_3.sh new file mode 100755 index 0000000000000000000000000000000000000000..b1e2539fa935bcb9ce69482325e1e4c6df0503f2 --- /dev/null +++ b/tests/pytest/pytest_3.sh @@ -0,0 +1,90 @@ +#!/bin/bash +ulimit -c unlimited + + +python3 ./test.py -f insert/randomNullCommit.py + +# user +python3 ./test.py -f user/user_create.py +python3 ./test.py -f user/pass_len.py + +# stable +python3 ./test.py -f stable/query_after_reset.py + +#query +python3 ./test.py -f query/filter.py +python3 ./test.py -f query/filterCombo.py +python3 ./test.py -f query/queryNormal.py +python3 ./test.py -f query/queryError.py +python3 ./test.py -f query/filterAllIntTypes.py +python3 ./test.py -f query/filterFloatAndDouble.py +python3 ./test.py -f query/filterOtherTypes.py +python3 ./test.py -f query/querySort.py +python3 ./test.py -f query/queryJoin.py +python3 ./test.py -f query/select_last_crash.py +python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py +python3 ./test.py -f query/queryConnection.py +python3 ./test.py -f query/queryCountCSVData.py +python3 ./test.py -f query/natualInterval.py +python3 ./test.py -f query/bug1471.py +#python3 ./test.py -f query/dataLossTest.py +python3 ./test.py -f query/bug1874.py +python3 ./test.py -f query/bug1875.py +python3 ./test.py -f query/bug1876.py +python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2118.py +python3 ./test.py -f query/bug2143.py +python3 ./test.py -f query/sliding.py +python3 ./test.py -f query/unionAllTest.py +python3 ./test.py -f query/bug2281.py +python3 ./test.py -f query/bug2119.py +python3 ./test.py -f query/isNullTest.py +python3 ./test.py -f query/queryWithTaosdKilled.py +python3 ./test.py -f query/floatCompare.py + +#stream +python3 ./test.py -f stream/metric_1.py +python3 ./test.py -f stream/metric_n.py +python3 ./test.py -f stream/new.py +python3 ./test.py -f stream/stream1.py +python3 ./test.py -f stream/stream2.py +#python3 ./test.py -f stream/parser.py +python3 ./test.py -f stream/history.py +python3 ./test.py -f stream/sys.py +python3 ./test.py -f stream/table_1.py +python3 ./test.py -f stream/table_n.py + +#alter table +python3 ./test.py -f alter/alter_table_crash.py + +# client +python3 ./test.py -f client/client.py +python3 ./test.py -f client/version.py +python3 ./test.py -f client/alterDatabase.py +python3 ./test.py -f client/noConnectionErrorTest.py + +# Misc +python3 testCompress.py +python3 testNoCompress.py +python3 testMinTablesPerVnode.py + + +python3 queryCount.py +python3 ./test.py -f query/queryGroupbyWithInterval.py +python3 client/twoClients.py +python3 test.py -f query/queryInterval.py +python3 test.py -f query/queryFillTest.py + +# tools +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdumpTest.py +python3 test.py -f tools/lowaTest.py +#python3 test.py -f tools/taosdemoTest2.py + +# subscribe +python3 test.py -f subscribe/singlemeter.py +#python3 test.py -f subscribe/stability.py +python3 test.py -f subscribe/supertable.py + diff --git a/tests/pytest/pytest_4.sh b/tests/pytest/pytest_4.sh new file mode 100755 index 0000000000000000000000000000000000000000..dada90c709a967df37693d7398692c6d7fc12039 --- /dev/null +++ b/tests/pytest/pytest_4.sh @@ -0,0 +1,39 @@ +python3 ./test.py -f update/merge_commit_data-0.py +# wal +python3 ./test.py -f wal/addOldWalTest.py + +# function +python3 ./test.py -f functions/all_null_value.py +# functions +python3 ./test.py -f functions/function_avg.py -r 1 +python3 ./test.py -f functions/function_bottom.py -r 1 +python3 ./test.py -f functions/function_count.py -r 1 +python3 ./test.py -f functions/function_diff.py -r 1 +python3 ./test.py -f functions/function_first.py -r 1 +python3 ./test.py -f functions/function_last.py -r 1 +python3 ./test.py -f functions/function_last_row.py -r 1 +python3 ./test.py -f functions/function_leastsquares.py -r 1 +python3 ./test.py -f functions/function_max.py -r 1 +python3 ./test.py -f functions/function_min.py -r 1 +python3 ./test.py -f functions/function_operations.py -r 1 +python3 ./test.py -f functions/function_percentile.py -r 1 +python3 ./test.py -f functions/function_spread.py -r 1 +python3 ./test.py -f functions/function_stddev.py -r 1 +python3 ./test.py -f functions/function_sum.py -r 1 +python3 ./test.py -f functions/function_top.py -r 1 +python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa_test2.py +python3 ./test.py -f functions/function_stddev_td2555.py +python3 ./test.py -f insert/metadataUpdate.py +python3 ./test.py -f tools/taosdemoTest2.py +python3 ./test.py -f query/last_cache.py +python3 ./test.py -f query/last_row_cache.py +python3 ./test.py -f account/account_create.py +python3 ./test.py -f alter/alter_table.py +python3 ./test.py -f query/queryGroupbySort.py + +python3 ./test.py -f insert/unsignedInt.py +python3 ./test.py -f insert/unsignedBigint.py +python3 ./test.py -f insert/unsignedSmallint.py +python3 ./test.py -f insert/unsignedTinyint.py +python3 ./test.py -f query/filterAllUnsignedIntTypes.py \ No newline at end of file diff --git a/tests/pytest/query/bug2118.py b/tests/pytest/query/bug2118.py new file mode 100644 index 0000000000000000000000000000000000000000..803f442a1b201b8fcd508e39f5f02b0ab95812aa --- /dev/null +++ b/tests/pytest/query/bug2118.py @@ -0,0 +1,52 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdLog.info("==========step1") + tdLog.info("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20))") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % + (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41))) + tdLog.info("==========step2") + tdLog.info("test percentile with group by normal_col ") + tdSql.query('select percentile(c1,1),percentile(c2,1),percentile(c6,1) from mt0 group by c3 limit 3 offset 2') + + tdSql.checkData(0,0,2.48) + tdSql.checkData(0,0,2.48) + tdSql.checkData(0,2,11.84) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/bug2119.py b/tests/pytest/query/bug2119.py new file mode 100644 index 0000000000000000000000000000000000000000..4224e55596c201f27fbdacd582a90ef5bfd45b85 --- /dev/null +++ b/tests/pytest/query/bug2119.py @@ -0,0 +1,41 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)") + + + print("==========step2") + print("query percentile from blank table") + tdSql.query('select percentile(c1,1) from t1') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/bug2281.py b/tests/pytest/query/bug2281.py new file mode 100644 index 0000000000000000000000000000000000000000..b8eb17f5cd8594389f0a218a95a7024e18735231 --- /dev/null +++ b/tests/pytest/query/bug2281.py @@ -0,0 +1,47 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)") + insertRows = 10 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into t1 values (%d , %d,%d)" % + (t0+i,i%100,i/2.0)) + + print("==========step2") + print("query diff && top") + tdSql.error('select diff(c1),top(c2) from t1') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/filterAllUnsignedIntTypes.py b/tests/pytest/query/filterAllUnsignedIntTypes.py new file mode 100644 index 0000000000000000000000000000000000000000..0ff2f2b6eeb1424b0bcfd9a9e54537e8f216f9c6 --- /dev/null +++ b/tests/pytest/query/filterAllUnsignedIntTypes.py @@ -0,0 +1,176 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.powers = [8, 16, 32, 64] + self.types = ["tinyint", "smallint", "int", "bigint"] + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + for i in range(len(self.powers)): + curType = self.types[i] + print("======= Verify filter for %s type =========" % (curType)) + tdLog.debug( + "create table st%s(ts timestamp, num %s unsigned) tags(id %s unsigned)" % + (curType, curType, curType)) + tdSql.execute( + "create table st%s(ts timestamp, num %s unsigned) tags(id %s unsigned)" % + (curType, curType, curType)) + + # create 10 tables, insert 10 rows for each table + for j in range(self.rowNum): + tdSql.execute( + "create table st%s%d using st%s tags(%d)" % + (curType, j + 1, curType, j + 1)) + for k in range(self.rowNum): + tdSql.execute( + "insert into st%s%d values(%d, %d)" % + (curType, j + 1, self.ts + k + 1, j * 10 + k + 1)) + + tdSql.error("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 11, pow(2, self.powers[i]) - 1 )) + tdSql.execute("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 12, pow(2, self.powers[i]) - 2 )) + tdSql.error("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 13, -1 )) + tdSql.execute("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 14, 0 )) + # select all data + # tdSql.query("select * from st%s " % curType) + # tdSql.checkRows(102) + + # > for int type on column + tdSql.query("select * from st%s where num > 50" % curType) + tdSql.checkRows(51) + + # >= for int type on column + tdSql.query("select * from st%s where num >= 50" % curType) + tdSql.checkRows(52) + + # = for int type on column + tdSql.query("select * from st%s where num = 50" % curType) + tdSql.checkRows(1) + + # < for int type on column + tdSql.query("select * from st%s where num < 50" % curType) + tdSql.checkRows(50) + + # <= for int type on column + tdSql.query("select * from st%s where num <= 50" % curType) + tdSql.checkRows(51) + + # <> for int type on column + tdSql.query("select * from st%s where num <> 50" % curType) + tdSql.checkRows(101) + + # != for int type on column + tdSql.query("select * from st%s where num != 50" % curType) + tdSql.checkRows(101) + + # range for int type on column + tdSql.query( + "select * from st%s where num > 50 and num < 100" % + curType) + tdSql.checkRows(49) + + tdSql.query( + "select * from st%s where num >= 50 and num < 100" % + curType) + tdSql.checkRows(50) + + tdSql.query( + "select * from st%s where num > 50 and num <= 100" % + curType) + tdSql.checkRows(50) + + tdSql.query( + "select * from st%s where num >= 50 and num <= 100" % + curType) + tdSql.checkRows(51) + + # > for int type on tag + tdSql.query("select * from st%s where id > 5" % curType) + tdSql.checkRows(52) + + # >= for int type on tag + tdSql.query("select * from st%s where id >= 5" % curType) + tdSql.checkRows(62) + + # = for int type on tag + tdSql.query("select * from st%s where id = 5" % curType) + tdSql.checkRows(10) + + # < for int type on tag + tdSql.query("select * from st%s where id < 5" % curType) + tdSql.checkRows(40) + + # <= for int type on tag + tdSql.query("select * from st%s where id <= 5" % curType) + tdSql.checkRows(50) + + # <> for int type on tag + tdSql.query("select * from st%s where id <> 5" % curType) + tdSql.checkRows(92) + + # != for int type on tag + tdSql.query("select * from st%s where id != 5" % curType) + tdSql.checkRows(92) + + # != for int type on tag + tdSql.query("select * from st%s where id != 5" % curType) + tdSql.checkRows(92) + + # range for int type on tag + tdSql.query("select * from st%s where id > 5 and id < 7" % curType) + tdSql.checkRows(10) + + tdSql.query( + "select * from st%s where id >= 5 and id < 7" % + curType) + tdSql.checkRows(20) + + tdSql.query( + "select * from st%s where id > 5 and id <= 7" % + curType) + tdSql.checkRows(20) + + tdSql.query( + "select * from st%s where id >= 5 and id <= 7" % + curType) + tdSql.checkRows(30) + + print( + "======= Verify filter for %s type finished =========" % + curType) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/floatCompare.py b/tests/pytest/query/floatCompare.py new file mode 100644 index 0000000000000000000000000000000000000000..cdbddb893e8c0af19fc4aee1596172ade759583f --- /dev/null +++ b/tests/pytest/query/floatCompare.py @@ -0,0 +1,172 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table st(ts timestamp, c1 float, c2 double) tags(t1 float, t2 double)") + tdSql.execute( + 'CREATE TABLE if not exists t1 using st tags(1.023, 3.00412)') + tdSql.execute( + 'CREATE TABLE if not exists t2 using st tags(2.005, 10.129001)') + + print("==============step2") + for i in range(10): + tdSql.execute("insert into t1 values(%d, %s, %s) t2 values(%d, %s, %s)" % + (self.ts + i, 1.0001 + i, 2.0001301 + i, self.ts + i, 1.0001 * i, 2.0001301 * i)) + + tdSql.query("select count(*) from st") + tdSql.checkData(0, 0, 20) + + tdSql.query("select count(*) from st where t1 > 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 < 1.023") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t1 >= 1.023") + tdSql.checkData(0, 0, 20) + + tdSql.query("select count(*) from st where t1 <= 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 = 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 <> 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 != 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 >= 1.023 and t1 <= 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 > 1.023 and t1 <= 1.023") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t1 >= 1.023 and t1 < 1.023") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t2 > 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 < 3.00412") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t2 >= 3.00412") + tdSql.checkData(0, 0, 20) + + tdSql.query("select count(*) from st where t2 <= 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 = 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 <> 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 != 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 >= 3.00412 and t2 <= 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 > 3.00412 and t2 <= 3.00412") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t2 >= 3.00412 and t2 < 3.00412") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t1 where c1 < 2.0001") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t1 where c1 <= 2.0001") + tdSql.checkData(0, 0, 2) + + tdSql.query("select count(*) from t1 where c1 = 2.0001") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t1 where c1 > 2.0001") + tdSql.checkData(0, 0, 8) + + tdSql.query("select count(*) from t1 where c1 >= 2.0001") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t1 where c1 <> 2.0001") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t1 where c1 != 2.0001") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t1 where c1 >= 2.0001 and c1 <= 2.0001") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t1 where c1 > 2.0001 and c1 <= 2.0001") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t1 where c1 >= 2.0001 and c1 < 2.0001") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t2 where c2 < 2.0001301") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t2 where c2 <= 2.0001301") + tdSql.checkData(0, 0, 2) + + tdSql.query("select count(*) from t2 where c2 = 2.0001301") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t2 where c2 > 2.0001301") + tdSql.checkData(0, 0, 8) + + tdSql.query("select count(*) from t2 where c2 >= 2.0001301") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t2 where c2 <> 2.0001301") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t2 where c2 != 2.0001301") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t2 where c2 >= 2.0001301 and c2 <= 2.0001301") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t2 where c2 > 2.0001301 and c2 <= 2.0001301") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t2 where c2 >= 2.0001301 and c2 < 2.0001301") + tdSql.checkRows(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/isNullTest.py b/tests/pytest/query/isNullTest.py new file mode 100644 index 0000000000000000000000000000000000000000..7b79679c7d9d9ac4629a69b32acb1a11b61a83c1 --- /dev/null +++ b/tests/pytest/query/isNullTest.py @@ -0,0 +1,128 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table st(ts timestamp, c1 int, c2 binary(20), c3 nchar(20)) tags(t1 int, t2 binary(20), t3 nchar(20))") + tdSql.execute("create table t1 using st tags(1, 'binary1', 'nchar1')") + tdSql.execute("insert into t2(ts, c2) using st(t2) tags('') values(%d, '')" % (self.ts + 10)) + tdSql.execute("insert into t3(ts, c2) using st(t3) tags('') values(%d, '')" % (self.ts + 10)) + + for i in range(10): + tdSql.execute("insert into t1 values(%d, %d, 'binary%d', 'nchar%d')" % (self.ts + i, i, i, i)) + tdSql.execute("insert into t2 values(%d, %d, 'binary%d', 'nchar%d')" % (self.ts + i, i, i, i)) + tdSql.execute("insert into t3 values(%d, %d, 'binary%d', 'nchar%d')" % (self.ts + i, i, i, i)) + + tdSql.execute("insert into t1(ts, c2) values(%d, '')" % (self.ts + 10)) + tdSql.execute("insert into t1(ts, c3) values(%d, '')" % (self.ts + 11)) + tdSql.execute("insert into t2(ts, c3) values(%d, '')" % (self.ts + 11)) + tdSql.execute("insert into t3(ts, c3) values(%d, '')" % (self.ts + 11)) + + tdSql.query("select count(*) from st") + tdSql.checkData(0, 0, 36) + + tdSql.query("select count(*) from st where t1 is null") + tdSql.checkData(0, 0, 24) + + tdSql.query("select count(*) from st where t1 is not null") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t2 is null") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t2 is not null") + tdSql.checkData(0, 0, 24) + + tdSql.error("select count(*) from st where t2 <> null") + tdSql.error("select count(*) from st where t2 = null") + + tdSql.query("select count(*) from st where t2 = '' ") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t2 <> '' ") + tdSql.checkData(0, 0, 24) + + tdSql.query("select count(*) from st where t3 is null") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t3 is not null") + tdSql.checkData(0, 0, 24) + + tdSql.error("select count(*) from st where t3 <> null") + tdSql.error("select count(*) from st where t3 = null") + + tdSql.query("select count(*) from st where t3 = '' ") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t3 <> '' ") + tdSql.checkData(0, 0, 24) + + tdSql.query("select count(*) from st where c1 is not null") + tdSql.checkData(0, 0, 30) + + tdSql.query("select count(*) from st where c1 is null") + tdSql.checkData(0, 0, 6) + + tdSql.query("select count(*) from st where c2 is not null") + tdSql.checkData(0, 0, 33) + + tdSql.query("select count(*) from st where c2 is null") + tdSql.checkData(0, 0, 3) + + tdSql.error("select count(*) from st where c2 <> null") + tdSql.error("select count(*) from st where c2 = null") + + tdSql.query("select count(*) from st where c2 = '' ") + tdSql.checkData(0, 0, 3) + + tdSql.query("select count(*) from st where c2 <> '' ") + tdSql.checkData(0, 0, 30) + + tdSql.query("select count(*) from st where c3 is not null") + tdSql.checkData(0, 0, 33) + + tdSql.query("select count(*) from st where c3 is null") + tdSql.checkData(0, 0, 3) + + tdSql.error("select count(*) from st where c3 <> null") + tdSql.error("select count(*) from st where c3 = null") + + tdSql.query("select count(*) from st where c3 = '' ") + tdSql.checkData(0, 0, 3) + + tdSql.query("select count(*) from st where c3 <> '' ") + tdSql.checkData(0, 0, 30) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/last_cache.py b/tests/pytest/query/last_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..c31d9821e2695dd7883674519325c24f0e473bf2 --- /dev/null +++ b/tests/pytest/query/last_cache.py @@ -0,0 +1,133 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.tables = 10 + self.rows = 20 + self.perfix = 't' + self.ts = 1601481600000 + + def insertData(self): + print("==============step1") + tdSql.execute("create table st (ts timestamp, c1 int) tags(t1 int)") + + for i in range(self.tables): + tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) + for j in range(self.rows): + tc = self.ts + j * 60000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, i, tc, j)) + + def executeQueries(self): + print("==============step2") + tdSql.query("select last(c1) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) from %s%d where ts <= %d" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) as b from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) from %s%d interval(1m)" % (self.perfix, 1)) + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from %s%d interval(1d)" % (self.perfix, 1)) + tdSql.checkData(0, 1, 19) + + tdSql.query("select last(c1) from %s%d where ts <= %d interval(1m)" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkRows(5) + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from st") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) as c from st where t1 <= 5") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where t1 <= 5 and ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) from st interval(1m)") + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from st interval(1d)") + tdSql.checkData(0, 1, 19) + + tdSql.query("select last(c1) from st group by t1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where ts <= %d interval(1m) group by t1" % (self.ts + 4 * 60000)) + tdSql.checkData(1, 1, 1) + tdSql.checkRows(50) + + def run(self): + tdSql.prepare() + + # last_cache_0.sim + tdSql.execute("create database test1 cachelast 0") + tdSql.execute("use test1") + self.insertData() + self.executeQueries() + + tdSql.execute("alter database test1 cachelast 1") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + tdSql.execute("alter database test1 cachelast 0") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + # last_cache_1.sim + tdSql.execute("create database test2 cachelast 1") + tdSql.execute("use test2") + self.insertData() + self.executeQueries() + + tdSql.execute("alter database test2 cachelast 0") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + tdSql.execute("alter database test2 cachelast 1") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..a0e81477096e9c846e109ae71020b40e47b39a84 --- /dev/null +++ b/tests/pytest/query/last_row_cache.py @@ -0,0 +1,194 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.tables = 10 + self.rows = 20 + self.columns = 50 + self.perfix = 't' + self.ts = 1601481600000 + + def insertData(self): + print("==============step1") + sql = "create table st(ts timestamp, " + for i in range(self.columns - 1): + sql += "c%d int, " % (i + 1) + sql += "c50 int) tags(t1 int)" + tdSql.execute(sql) + + for i in range(self.tables): + tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) + for j in range(self.rows): + tc = self.ts + j * 60000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, i, tc, j)) + + def executeQueries(self): + print("==============step2") + tdSql.query("select last_row(c1) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) from %s%d where ts <= %d" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as b from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) from st") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) as c from st where ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as c from st where t1 < 5") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) as c from st where t1 <= 5 and ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as c from st group by t1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 19) + + tc = self.ts + 1 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, 10)) + + tc = self.ts + 3 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) + + tc = self.ts + 5 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, -1)) + + tc = self.ts + 7 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) + + def insertData2(self): + tc = self.ts + 1 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, 10)) + + tc = self.ts + 3 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) + + tc = self.ts + 5 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, -1)) + + tc = self.ts + 7 * 3600000 + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) + + def executeQueries2(self): + # For stable + tc = self.ts + 6 * 3600000 + tdSql.query("select last_row(c1) from st where ts < %d " % tc) + tdSql.checkData(0, 0, -1) + + tc = self.ts + 8 * 3600000 + tdSql.query("select last_row(*) from st where ts < %d " % tc) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(*) from st") + tdSql.checkData(0, 1, None) + + tc = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts < %d " % tc) + tdSql.checkData(0, 1, None) + + tc1 = self.ts + 1 * 3600000 + tc2 = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts > %d and ts <= %d" % (tc1, tc2)) + tdSql.checkData(0, 1, None) + + # For table + tc = self.ts + 6 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, -1) + + tc = self.ts + 8 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(*) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 1, None) + + tc = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, None) + + tc1 = self.ts + 1 * 3600000 + tc2 = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts > %d and ts <= %d" % (tc1, tc2)) + tdSql.checkData(0, 1, None) + + def run(self): + tdSql.prepare() + + print("============== last_row_cache_0.sim") + tdSql.execute("create database test1 cachelast 0") + tdSql.execute("use test1") + self.insertData() + self.executeQueries() + self.insertData2() + self.executeQueries2() + + print("============== alter last cache") + tdSql.execute("alter database test1 cachelast 1") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test1 cachelast 0") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + print("============== last_row_cache_1.sim") + tdSql.execute("create database test2 cachelast 1") + tdSql.execute("use test2") + self.insertData() + self.executeQueries() + self.insertData2() + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 0") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 1") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index da0ef96d0efde77b716ded06cc7efaa96ea3bb7b..8cec38780e5e873cb07d8ade45ef56b5d4ee43f7 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -16,13 +16,15 @@ import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql - +from util.dnodes import tdDnodes class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + self.ts = 1538548685000 + def run(self): tdSql.prepare() @@ -44,6 +46,83 @@ class TDTestCase: tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'") tdSql.checkRows(1) + tdSql.query("select tbname, dev from dev_001") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'dev_001') + tdSql.checkData(0, 1, 'dev_01') + + tdSql.query("select tbname, dev, tagtype from dev_001") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 'dev_001') + tdSql.checkData(0, 1, 'dev_01') + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, 'dev_001') + tdSql.checkData(1, 1, 'dev_01') + tdSql.checkData(1, 2, 1) + + ## test case for https://jira.taosdata.com:18080/browse/TD-2488 + tdSql.execute("create table m1(ts timestamp, k int) tags(a int)") + tdSql.execute("create table t1 using m1 tags(1)") + tdSql.execute("create table t2 using m1 tags(2)") + tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)") + tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)") + tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)") + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + ## test case for https://jira.taosdata.com:18080/browse/TD-1930 + tdSql.execute("create table tb(ts timestamp, c1 int, c2 binary(10), c3 nchar(10), c4 float, c5 bool)") + for i in range(10): + tdSql.execute("insert into tb values(%d, %d, 'binary%d', 'nchar%d', %f, %d)" % (self.ts + i, i, i, i, i + 0.1, i % 2)) + + tdSql.error("select * from tb where c2 = binary2") + tdSql.error("select * from tb where c3 = nchar2") + + tdSql.query("select * from tb where c2 = 'binary2' ") + tdSql.checkRows(1) + + tdSql.query("select * from tb where c3 = 'nchar2' ") + tdSql.checkRows(1) + + tdSql.query("select * from tb where c1 = '2' ") + tdSql.checkRows(1) + + tdSql.query("select * from tb where c1 = 2 ") + tdSql.checkRows(1) + + tdSql.query("select * from tb where c4 = '0.1' ") + tdSql.checkRows(1) + + tdSql.query("select * from tb where c4 = 0.1 ") + tdSql.checkRows(1) + + tdSql.query("select * from tb where c5 = true ") + tdSql.checkRows(5) + + tdSql.query("select * from tb where c5 = 'true' ") + tdSql.checkRows(5) + + # For jira: https://jira.taosdata.com:18080/browse/TD-2850 + tdSql.execute("create database 'Test' ") + tdSql.execute("use 'Test' ") + tdSql.execute("create table 'TB'(ts timestamp, 'Col1' int) tags('Tag1' int)") + tdSql.execute("insert into 'Tb0' using tb tags(1) values(now, 1)") + tdSql.query("select * from tb") + tdSql.checkRows(1) + + tdSql.query("select * from tb0") + tdSql.checkRows(1) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryError.py b/tests/pytest/query/queryError.py index f1fd9c0dec480d8d4e0b49635fd23b275eeeaa24..539ce5141fae9cfba7ba63b569179c0170ef428d 100644 --- a/tests/pytest/query/queryError.py +++ b/tests/pytest/query/queryError.py @@ -56,6 +56,15 @@ class TDTestCase: # query .. order by non-time field tdSql.error("select * from st order by name") + # TD-2133 + tdSql.error("select diff(tagtype),bottom(tagtype,1) from dev_001") + + # TD-2190 + tdSql.error("select min(tagtype),max(tagtype) from dev_002 interval(1n) fill(prev)") + + # TD-2208 + tdSql.error("select diff(tagtype),top(tagtype,1) from dev_001") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryFillTest.py b/tests/pytest/query/queryFillTest.py index 9fd898041a525f9aaba65bda1233f94b1bb922e6..e50d02faf27f33191f4a76c535443f6802e0ed28 100644 --- a/tests/pytest/query/queryFillTest.py +++ b/tests/pytest/query/queryFillTest.py @@ -47,11 +47,38 @@ class TDTestCase: tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h)") tdSql.checkRows(139) + tdSql.checkData(0, 1, -1.5) + tdSql.checkData(138, 1, -1.0) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(none)") + tdSql.checkRows(139) + tdSql.checkData(0, 1, -1.5) + tdSql.checkData(138, 1, -1.0) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(value, 2.0)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(140, 1, 2.0) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(prev)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, None) + tdSql.checkData(140, 1, -1.0) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(null)") tdSql.checkRows(141) tdSql.checkData(0, 1, None) - tdSql.checkData(140, 1, None) + tdSql.checkData(140, 1, None) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(linear)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, None) + tdSql.checkData(140, 1, None) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(next)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, -1.5) + tdSql.checkData(140, 1, None) tdSql.query("select max(col1) - min(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' and id = 1 group by loc, id") rows = tdSql.queryRows diff --git a/tests/pytest/query/queryGroupbySort.py b/tests/pytest/query/queryGroupbySort.py index 28ef897f42924d3e41c6598cac6b4c51e33d38ef..063db936087c125e45051da0094b57a9fd184b9b 100644 --- a/tests/pytest/query/queryGroupbySort.py +++ b/tests/pytest/query/queryGroupbySort.py @@ -28,19 +28,35 @@ class TDTestCase: def run(self): tdSql.prepare() - tdSql.execute( - "create table stb(ts timestamp,i int) tags (p_id nchar(20));") - tdSql.execute( - "insert into tb using stb tags('11231') values (%d, %d) (%d, %d) (%d, %d) (%d, %d)" - % (self.ts, 12, self.ts + 1, 15, self.ts + 2, 15, self.ts + 3, 12)) + tdSql.execute("CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int, t3 float, t4 double)") + tdSql.execute("CREATE TABLE D1001 USING meters TAGS ('Beijing.Chaoyang', 2 , NULL, NULL)") + tdSql.execute("CREATE TABLE D1002 USING meters TAGS ('Beijing.Chaoyang', 3 , NULL , 1.7)") + tdSql.execute("CREATE TABLE D1003 USING meters TAGS ('Beijing.Chaoyang', 3 , 1.1 , 1.7)") + tdSql.execute("INSERT INTO D1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) (1538548696800, 12.3, 221, 0.31)") + tdSql.execute("INSERT INTO D1002 VALUES (1538548685001, 10.5, 220, 0.28) (1538548696800, 12.3, 221, 0.31)") + tdSql.execute("INSERT INTO D1003 VALUES (1538548685001, 10.5, 220, 0.28) (1538548696800, 12.3, 221, 0.31)") + tdSql.query("SELECT SUM(current), AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by ts DESC") + tdSql.checkRows(3) + tdSql.checkData(0, 0, "2018-10-03 14:38:16") + tdSql.checkData(1, 0, "2018-10-03 14:38:15") + tdSql.checkData(2, 0, "2018-10-03 14:38:05") - tdSql.query(''' select last(ts) p_time,i from stb where p_id='11231' and ts>=%d and ts <=%d - group by i order by time desc limit 100 ''' % (self.ts, self.ts + 4)) - tdSql.checkRows(2) - tdSql.checkData(0, 0, "2018-09-17 09:00:00.003000") - tdSql.checkData(1, 0, "2018-09-17 09:00:00.002000") + tdSql.query("SELECT SUM(current), AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by ts ASC") + tdSql.checkRows(3) + tdSql.checkData(0, 0, "2018-10-03 14:38:05") + tdSql.checkData(1, 0, "2018-10-03 14:38:15") + tdSql.checkData(2, 0, "2018-10-03 14:38:16") + tdSql.error("SELECT SUM(current) as s, AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by s ASC") + tdSql.error("SELECT SUM(current) as s, AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by s DESC") + + #add for TD-3170 + tdSql.query("select avg(current) from meters group by t3;") + tdSql.checkData(0, 0, 11.6) + tdSql.query("select avg(current) from meters group by t4;") + tdSql.query("select avg(current) from meters group by t3,t4;") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 871c076c0813f0ccd340d33bff992401e522df4a..ce8d05ae50aa61646b20e07e9484d00559e92b49 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -24,7 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.ts = 1593548685000 + self.ts = 1593548685000 def run(self): tdSql.prepare() @@ -84,7 +84,39 @@ class TDTestCase: tdDnodes.start(1) tdSql.query("select last(*) from t interval(1s)") tdSql.checkRows(10000) + + # test case for https://jira.taosdata.com:18080/browse/TD-2601 + newTs = 1601481600000 + + tdSql.execute("create database test2") + tdSql.execute("use test2") + tdSql.execute("create table t (ts timestamp, voltage int)") + for i in range(100): + tdSql.execute("insert into t values(%d, %d)" % (newTs + i * 10000000, i)) + tdSql.query("select sum(voltage) from t where ts >='2020-10-01 00:00:00' and ts <='2020-12-01 00:00:00' interval(1n) fill(NULL)") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 4950) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, None) + + # test case for https://jira.taosdata.com:18080/browse/TD-2659， https://jira.taosdata.com:18080/browse/TD-2660 + tdSql.execute("create database test3") + tdSql.execute("use test3") + tdSql.execute("create table tb(ts timestamp, c int)") + tdSql.execute("insert into tb values('2020-10-30 18:11:56.680', -111)") + tdSql.execute("insert into tb values('2020-11-19 18:11:45.773', null)") + tdSql.execute("insert into tb values('2020-12-09 18:11:17.098', null)") + tdSql.execute("insert into tb values('2020-12-29 11:00:49.412', 1)") + tdSql.execute("insert into tb values('2020-12-29 11:00:50.412', 2)") + tdSql.execute("insert into tb values('2020-12-29 11:00:52.412', 3)") + + tdSql.query("select first(ts),twa(c) from tb interval(14a)") + tdSql.checkRows(6) + + tdSql.query("select twa(c) from tb group by c") + tdSql.checkRows(4) + def stop(self): tdSql.close() diff --git a/tests/pytest/query/queryNullValueTest.py b/tests/pytest/query/queryNullValueTest.py index bc0b11827e18b0b436edac1cee2f04bafb05fab3..5a95b369fe39d6bc1216847b3f108dff7d7171b7 100644 --- a/tests/pytest/query/queryNullValueTest.py +++ b/tests/pytest/query/queryNullValueTest.py @@ -25,7 +25,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.types = ["tinyint", "smallint", "int", "bigint", "float", "double", "bool", "binary(10)", "nchar(10)"] + self.types = ["tinyint", "smallint", "int", "bigint", "float", "double", "bool", "binary(10)", "nchar(10)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] self.ts = 1537146000000 def checkNullValue(self, result): @@ -50,7 +50,7 @@ class TDTestCase: tdSql.execute("insert into t0 values (%d, NULL)" % (self.ts)) tdDnodes.stop(1) - tdLog.sleep(10) + # tdLog.sleep(10) tdDnodes.start(1) tdSql.execute("use db") tdSql.query("select * from t0") @@ -62,7 +62,7 @@ class TDTestCase: tdSql.execute("create table t1 (ts timestamp, col %s)" % self.types[i]) tdSql.execute("insert into t1 values (%d, NULL)" % (self.ts)) tdDnodes.stop(1) - tdLog.sleep(10) + # tdLog.sleep(10) tdDnodes.start(1) tdSql.execute("use db") diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py index 72af38450c862f7558df313dd53316599a23dedd..720ae745cb9b3780f1ca7ffaf96d76eda5f307b1 100644 --- a/tests/pytest/query/queryPerformance.py +++ b/tests/pytest/query/queryPerformance.py @@ -16,10 +16,16 @@ import sys import os import taos import time +import argparse class taosdemoQueryPerformace: - def initConnection(self): + def __init__(self, clearCache, commitID, dbName, stbName, tbPerfix): + self.clearCache = clearCache + self.commitID = commitID + self.dbName = dbName + self.stbName = stbName + self.tbPerfix = tbPerfix self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -30,92 +36,109 @@ class taosdemoQueryPerformace: self.password, self.config) + def createPerfTables(self): + cursor = self.conn.cursor() + cursor.execute("create database if not exists %s" % self.dbName) + cursor.execute("use %s" % self.dbName) + cursor.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50)) tags(query_id int, query_sql binary(300))" % self.stbName) + + sql = "select count(*) from test.meters" + tableid = 1 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.meters" + tableid = 2 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select count(*) from test.meters where loc='beijing'" + tableid = 3 + cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.meters where areaid=10" + tableid = 4 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.t10 interval(10s)" + tableid = 5 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select last_row(*) from meters" + tableid = 6 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select * from meters" + tableid = 7 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'" + tableid = 8 + cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + + cursor.close() def query(self): - cursor = self.conn.cursor() - cursor.execute("use test") - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select count(*) from test.meters") - totalTime += time.time() - startTime - print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from test.meters") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from test.meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select count(*) from test.meters where loc='beijing'") - totalTime += time.time() - startTime - print("query time for: select count(*) from test.meters where loc='beijing' %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from test.meters where areaid=10 %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from test.t10 interval(10s) %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select last_row(*) from meters") - totalTime += time.time() - startTime - print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select * from meters") - totalTime += time.time() - startTime - print("query time for: select * from meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100)) + cursor = self.conn.cursor() + print("==================== query performance ====================") + + cursor.execute("use %s" % self.dbName) + cursor.execute("select tbname, query_id, query_sql from %s" % self.stbName) + + for data in cursor: + table_name = data[0] + query_id = data[1] + sql = data[2] + + totalTime = 0 + cursor2 = self.conn.cursor() + cursor2.execute("use test") + for i in range(100): + if(self.clearCache == True): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + + startTime = time.time() + cursor2.execute(sql) + totalTime += time.time() - startTime + cursor2.close() + print("query time for: %s %f seconds" % (sql, totalTime / 100)) + + cursor3 = self.conn.cursor() + cursor3.execute("insert into %s.%s values(now, %f, '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID)) + + cursor3.close() + cursor.close() if __name__ == '__main__': - perftest = taosdemoQueryPerformace() - perftest.initConnection() - perftest.query() \ No newline at end of file + parser = argparse.ArgumentParser() + parser.add_argument( + '-r', + '--remove-cache', + action='store_true', + default=False, + help='clear cache before query (default: False)') + parser.add_argument( + '-c', + '--commit-id', + action='store', + default='null', + type=str, + help='git commit id (default: null)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='perf', + type=str, + help='Database name to be created (default: perf)') + parser.add_argument( + '-t', + '--stable-name', + action='store', + default='query_tb', + type=str, + help='table name to be created (default: query_tb)') + parser.add_argument( + '-p', + '--table-perfix', + action='store', + default='q', + type=str, + help='table name perfix (default: q)') + + args = parser.parse_args() + perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix) + perftest.createPerfTables() + perftest.query() diff --git a/tests/pytest/query/querySort.py b/tests/pytest/query/querySort.py index 649e0dc1cb3191ba08b3f2da0a5edee3afc66575..17022bdc41057bcb67e1530a2cb6d399bada20ff 100644 --- a/tests/pytest/query/querySort.py +++ b/tests/pytest/query/querySort.py @@ -73,19 +73,19 @@ class TDTestCase: print("======= step 1: create table and insert data =========") tdLog.debug( ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double, - tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, - tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''') + tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20), tbcol11 tinyint unsigned, tbcol12 smallint unsigned, tbcol13 int unsigned, tbcol14 bigint unsigned) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, + tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20), tagcol11 tinyint unsigned, tagcol12 smallint unsigned, tagcol13 int unsigned, tagcol14 bigint unsigned)''') tdSql.execute( ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double, - tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, - tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''') + tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20), tbcol11 tinyint unsigned, tbcol12 smallint unsigned, tbcol13 int unsigned, tbcol14 bigint unsigned) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, + tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20), tagcol11 tinyint unsigned, tagcol12 smallint unsigned, tagcol13 int unsigned, tagcol14 bigint unsigned)''') for i in range(self.rowNum): - tdSql.execute("create table st%d using st tags(%d, %d, %d, %d, %f, %f, %d, 'tag%d', '标签%d')" % ( - i + 1, i + 1, i + 1, i + 1, i + 1, 1.1 * (i + 1), 1.23 * (i + 1), (i + 1) % 2, i + 1, i + 1)) + tdSql.execute("create table st%d using st tags(%d, %d, %d, %d, %f, %f, %d, 'tag%d', '标签%d', %d, %d, %d, %d)" % ( + i + 1, i + 1, i + 1, i + 1, i + 1, 1.1 * (i + 1), 1.23 * (i + 1), (i + 1) % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) for j in range(self.rowNum): - tdSql.execute("insert into st%d values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" % ( - i + 1, self.ts + 10 * (i + 1) + j + 1, j + 1, j + 1, j + 1, j + 1, 1.1 * (j + 1), 1.23 * (j + 1), (j + 1) % 2, j + 1, j + 1)) + tdSql.execute("insert into st%d values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % ( + i + 1, self.ts + 10 * (i + 1) + j + 1, j + 1, j + 1, j + 1, j + 1, 1.1 * (j + 1), 1.23 * (j + 1), (j + 1) % 2, j + 1, j + 1, i + 1, i + 1, i + 1, i + 1)) print("======= step 2: verify order for each column =========") # sort for timestamp in asc order diff --git a/tests/pytest/query/queryWithTaosdKilled.py b/tests/pytest/query/queryWithTaosdKilled.py new file mode 100644 index 0000000000000000000000000000000000000000..28f9b87636987559669952a5fa88c25963fa9388 --- /dev/null +++ b/tests/pytest/query/queryWithTaosdKilled.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + self.conn = conn + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def createOldDir(self): + path = tdDnodes.dnodes[1].getDnodeRootDir(1) + print(path) + tdLog.info("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) + os.system("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) + + def run(self): + # os.system("rm -rf %s/ " % tdDnodes.getDnodesRootDir()) + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, speed int)") + tdSql.execute("insert into st values(now, 1)") + tdSql.query("select count(*) from st") + tdSql.checkRows(1) + + + self.createOldDir() + tdLog.sleep(10) + + print("force kill taosd") + os.system("sudo kill -9 $(pgrep -x taosd)") + os.system("") + tdDnodes.start(1) + + tdSql.init(self.conn.cursor()) + tdSql.execute("use db") + tdSql.query("select count(*) from st") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/sliding.py b/tests/pytest/query/sliding.py index 810d90117a73bba81ea010f81aa605196a496dc4..d7a7c9a36964796dd530f913523f8f4d344d467a 100644 --- a/tests/pytest/query/sliding.py +++ b/tests/pytest/query/sliding.py @@ -52,7 +52,15 @@ class TDTestCase: tdSql.query("select count(*) from meters group by loc") tdSql.checkRows(2) - tdSql.error("select * from meters group by loc sliding(5s)") + tdSql.error("select * from meters group by loc sliding(5s)") + + # Fix defect: https://jira.taosdata.com:18080/browse/TD-2700 + tdSql.execute("create database test") + tdSql.execute("use test") + tdSql.execute("create table t1(ts timestamp, k int)") + tdSql.execute("insert into t1 values(1500000001000, 0)") + tdSql.query("select sum(k) from t1 interval(1d) sliding(1h)") + tdSql.checkRows(24) def stop(self): tdSql.close() diff --git a/tests/pytest/query/unionAllTest.py b/tests/pytest/query/unionAllTest.py new file mode 100644 index 0000000000000000000000000000000000000000..1b69c8ac4d015a2ad8db72947022b7bd27e92756 --- /dev/null +++ b/tests/pytest/query/unionAllTest.py @@ -0,0 +1,65 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1500000000000 + self.num = 10 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 int) tags(loc nchar(20))") + tdSql.execute("create table t0 using st tags('nchar0')") + tdSql.execute("create table t1 using st tags('nchar1')") + tdSql.execute("create table t2 using st tags('nchar2')") + tdSql.execute("create table t3 using st tags('nchar3')") + tdSql.execute("create table t4 using st tags('nchar4')") + tdSql.execute("create table t5 using st tags('nchar5')") + + for i in range(self.num): + tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i)) + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i)) + tdSql.execute("insert into t2 values(%d, %d)" % (self.ts + i, i)) + tdSql.execute("insert into t3 values(%d, %d)" % (self.ts + i, i)) + tdSql.execute("insert into t4 values(%d, %d)" % (self.ts + i, i)) + tdSql.execute("insert into t5 values(%d, %d)" % (self.ts + i, i)) + + sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1 + union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1''' + tdSql.query(sql) + tdSql.checkRows(5) + + sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1 + union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5' limit 1''' + tdSql.query(sql) + tdSql.checkRows(6) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/stable/insert.py b/tests/pytest/stable/insert.py index 3d37e6726c965a8963f728fdd1d8dc89e1cee339..0ef816da8d02fc6dcb48954435aad51342248e56 100644 --- a/tests/pytest/stable/insert.py +++ b/tests/pytest/stable/insert.py @@ -47,6 +47,40 @@ class TDTestCase: tdSql.query("select * from db.st where dev='dev_02'") tdSql.checkRows(1) + #For: https://jira.taosdata.com:18080/browse/TD-2671 + print("==============step3") + tdSql.execute( + "create stable if not exists stb (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_01 using stb tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_02 using stb tags("dev_02")') + + print("==============step4") + + tdSql.execute( + """INSERT INTO dev_01(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_02 VALUES('2020-05-13 10:00:00.001', 1)""") + + tdSql.query("select * from db.stb where dev='dev_01'") + tdSql.checkRows(2) + + tdSql.query("select * from db.stb where dev='dev_02'") + tdSql.checkRows(1) + + tdSql.query("describe db.stb") + tdSql.checkRows(3) + + tdSql.execute("alter stable db.stb add tag t1 int") + tdSql.query("describe db.stb") + tdSql.checkRows(4) + + tdSql.execute("drop stable db.stb") + tdSql.query("show stables") + tdSql.checkRows(1) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/stream/history.py b/tests/pytest/stream/history.py index 890580001cf40a13ba18f6f81a3bd0faaac67991..cb8a4d598651473f907aa05a0609c9ce68c78f82 100644 --- a/tests/pytest/stream/history.py +++ b/tests/pytest/stream/history.py @@ -48,12 +48,16 @@ class TDTestCase: tdSql.execute("insert into car3 values('2019-01-01 00:00:01.389', 1)") tdSql.execute("insert into car4 values('2019-01-01 00:00:01.829', 1)") + tdSql.error("create table strm as select count(*) from cars") + tdSql.execute("create table strm as select count(*) from cars interval(4s)") tdSql.waitedQuery("select * from strm", 2, 100) tdSql.checkData(0, 1, 11) tdSql.checkData(1, 1, 2) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/stream/metric_n.py b/tests/pytest/stream/metric_n.py new file mode 100644 index 0000000000000000000000000000000000000000..d223fe81fc79835047bac8ca2341cdbeac2e6617 --- /dev/null +++ b/tests/pytest/stream/metric_n.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tbNum = 10 + rowNum = 20 + totalNum = tbNum * rowNum + + tdSql.prepare() + + tdLog.info("===== preparing data =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step 1 =====") + tdSql.query("select count(*), count(tbcol), count(tbcol2) from stb interval(1d)") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 2 =====") + tdSql.execute("create table strm_c3 as select count(*), count(tbcol), count(tbcol2) from stb interval(1d)") + + tdLog.info("===== step 3 =====") + tdSql.execute("create table strm_c32 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + + tdLog.info("===== step 4 =====") + tdSql.query("select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 5 =====") + tdSql.execute("create table strm_c31 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + + tdLog.info("===== step 6 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.execute("create table strm_avg as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb interval(1d)") + + tdLog.info("===== step 7 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from stb where ts < now + 4m interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, totalNum) + + tdLog.info("===== step 8 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from stb where ts < now + 4m interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, totalNum) + + tdLog.info("===== step 9 =====") + tdSql.waitedQuery("select * from strm_c3", 1, 120) + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 10 =====") + tdSql.waitedQuery("select * from strm_c31", 1, 30) + for i in range(1, 10): + tdSql.checkData(0, i, totalNum) + + tdLog.info("===== step 11 =====") + tdSql.waitedQuery("select * from strm_avg", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/new.py b/tests/pytest/stream/new.py index 12ec6d4507710869632eac77d719217b3b0ed7b3..70f300e937b2ed422231a774d2bca1183ce6fd0a 100644 --- a/tests/pytest/stream/new.py +++ b/tests/pytest/stream/new.py @@ -43,7 +43,7 @@ class TDTestCase: tdLog.info("=============== step3") start = time.time() tdSql.waitedQuery("select * from st", 1, 120) - delay = int(time.time() - start) + 20 + delay = int(time.time() - start) + 80 v = tdSql.getData(0, 3) if v >= 51: tdLog.exit("value is %d, which is larger than 51" % v) diff --git a/tests/pytest/stream/stream2.py b/tests/pytest/stream/stream2.py index 44882f59721c4967ebb3fea12e356bb7e95d71ae..d71742048a45774fbe6909093840f265b829eb4c 100644 --- a/tests/pytest/stream/stream2.py +++ b/tests/pytest/stream/stream2.py @@ -87,6 +87,10 @@ class TDTestCase: tdSql.checkData(0, 3, rowNum) except Exception as e: tdLog.info(repr(e)) + + tdSql.query("show streams") + tdSql.checkRows(1) + tdSql.checkData(0, 2, 's0') tdLog.info("===== step8 =====") tdSql.query( @@ -142,6 +146,12 @@ class TDTestCase: except Exception as e: tdLog.info(repr(e)) + tdSql.query("show streams") + tdSql.checkRows(2) + tdSql.checkData(0, 2, 's1') + tdSql.checkData(1, 2, 's0') + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/stream/stream3.py b/tests/pytest/stream/stream3.py new file mode 100644 index 0000000000000000000000000000000000000000..9a5c6c9aeca08bff1c94861255919255eef89100 --- /dev/null +++ b/tests/pytest/stream/stream3.py @@ -0,0 +1,108 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + ts = 1500000000000 + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute( + "create table stb0(ts timestamp, col1 binary(20), col2 nchar(20)) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb0 tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (%d, 'binary%d', 'nchar%d')" % + (i, ts + 60000 * j, j, j)) + tdSql.execute("insert into tb0 values(%d, null, null)" % (ts + 10000000)) + time.sleep(0.1) + + tdLog.info("===== step2 =====") + tdSql.query( + "select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + + tdSql.query("show tables") + tdSql.checkRows(tbNum) + tdSql.execute( + "create table s0 as select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step3 =====") + tdSql.waitedQuery("select * from s0", 1, 120) + try: + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + except Exception as e: + tdLog.info(repr(e)) + + tdLog.info("===== step4 =====") + tdSql.execute("drop table s0") + tdSql.query("show tables") + tdSql.checkRows(tbNum) + + tdLog.info("===== step5 =====") + tdSql.error("select * from s0") + + tdLog.info("===== step6 =====") + time.sleep(0.1) + tdSql.execute( + "create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step7 =====") + tdSql.waitedQuery("select * from s0", 1, 120) + try: + tdSql.checkData(0, 1, rowNum + 1) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + except Exception as e: + tdLog.info(repr(e)) + + tdLog.info("===== step8 =====") + tdSql.query( + "select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/sys.py b/tests/pytest/stream/sys.py new file mode 100644 index 0000000000000000000000000000000000000000..a73e7043e8c65b2eb9c78fbcb99d4e546ddf9ae4 --- /dev/null +++ b/tests/pytest/stream/sys.py @@ -0,0 +1,62 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# migrated from 'stream_on_sys.sim' +# -*- coding: utf-8 -*- +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + updatecfgDict = {'monitor': 1} + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def run(self): + time.sleep(5) + tdSql.execute("use log") + + tdSql.execute("create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s)") + tdSql.execute("create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s)") + tdSql.execute("create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s)") + tdSql.execute("create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s)") + tdSql.execute("create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s)") + tdSql.execute("create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s)") + + sqls = [ + "select * from cpustrm", + "select * from memstrm", + "select * from diskstrm", + "select * from bandstrm", + "select * from reqstrm", + "select * from iostrm", + ] + for sql in sqls: + (rows, _) = tdSql.waitedQuery(sql, 1, 120) + if rows < 1: + tdLog.exit("failed: sql:%s, expect at least one row" % sql) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/stream/table_1.py b/tests/pytest/stream/table_1.py new file mode 100644 index 0000000000000000000000000000000000000000..a9fd57393112177fe0e290cadeaabd97d669daca --- /dev/null +++ b/tests/pytest/stream/table_1.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def createFuncStream(self, expr, suffix, value): + tbname = "strm_" + suffix + tdLog.info("create stream table %s" % tbname) + tdSql.query("select %s from tb1 interval(1d)" % expr) + tdSql.checkData(0, 1, value) + tdSql.execute("create table %s as select %s from tb1 interval(1d)" % (tbname, expr)) + + def checkStreamData(self, suffix, value): + sql = "select * from strm_" + suffix + tdSql.waitedQuery(sql, 1, 120) + tdSql.checkData(0, 1, value) + + def run(self): + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + self.createFuncStream("count(*)", "c1", rowNum) + self.createFuncStream("count(tbcol)", "c2", rowNum) + self.createFuncStream("count(tbcol2)", "c3", rowNum) + self.createFuncStream("avg(tbcol)", "av", 9.5) + self.createFuncStream("sum(tbcol)", "su", 190) + self.createFuncStream("min(tbcol)", "mi", 0) + self.createFuncStream("max(tbcol)", "ma", 19) + self.createFuncStream("first(tbcol)", "fi", 0) + self.createFuncStream("last(tbcol)", "la", 19) + self.createFuncStream("stddev(tbcol)", "st", 5.766281297335398) + self.createFuncStream("percentile(tbcol, 1)", "pe", 0.19) + self.createFuncStream("count(tbcol)", "as", rowNum) + + self.checkStreamData("c1", rowNum) + self.checkStreamData("c2", rowNum) + self.checkStreamData("c3", rowNum) + self.checkStreamData("av", 9.5) + self.checkStreamData("su", 190) + self.checkStreamData("mi", 0) + self.checkStreamData("ma", 19) + self.checkStreamData("fi", 0) + self.checkStreamData("la", 19) + self.checkStreamData("st", 5.766281297335398) + self.checkStreamData("pe", 0.19) + self.checkStreamData("as", rowNum) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/table_n.py b/tests/pytest/stream/table_n.py new file mode 100644 index 0000000000000000000000000000000000000000..371af769778bce1eb1e6cf1bac89333006c582a8 --- /dev/null +++ b/tests/pytest/stream/table_n.py @@ -0,0 +1,143 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== preparing data =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step 1 =====") + tdSql.query("select count(*), count(tbcol), count(tbcol2) from tb1 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 2 =====") + tdSql.execute("create table strm_c3 as select count(*), count(tbcol), count(tbcol2) from tb1 interval(1d)") + + tdLog.info("===== step 3 =====") + tdSql.execute("create table strm_c32 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + + tdLog.info("===== step 4 =====") + tdSql.query("select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 5 =====") + tdSql.execute("create table strm_c31 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + + tdLog.info("===== step 6 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from tb1 interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.execute("create table strm_avg as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from tb1 interval(1d)") + + tdLog.info("===== step 7 =====") + tdSql.query("select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from tb1 interval(1d)") + tdSql.checkData(0, 1, 5.766281297335398) + tdSql.checkData(0, 3, 0.19) + tdSql.execute("create table strm_ot as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from tb1 interval(1d)") + + tdLog.info("===== step 8 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, 5.766281297335398) + tdSql.checkData(0, 8, 0.19) + tdSql.checkData(0, 9, rowNum) + tdSql.execute("create table strm_to as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 interval(1d)") + + tdLog.info("===== step 9 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 where ts < now + 4m interval(1d)") + tdSql.checkData(0, 9, rowNum) + tdSql.execute("create table strm_wh as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 where ts < now + 4m interval(1d)") + + tdLog.info("===== step 10 =====") + tdSql.waitedQuery("select * from strm_c3", 1, 120) + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 11 =====") + tdSql.waitedQuery("select * from strm_c31", 1, 30) + for i in range(1, 10): + tdSql.checkData(0, i, rowNum) + + tdLog.info("===== step 12 =====") + tdSql.waitedQuery("select * from strm_avg", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + + tdLog.info("===== step 13 =====") + tdSql.waitedQuery("select * from strm_ot", 1, 20) + tdSql.checkData(0, 1, 5.766281297335398) + tdSql.checkData(0, 3, 0.19) + + tdLog.info("===== step 14 =====") + tdSql.waitedQuery("select * from strm_to", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, 5.766281297335398) + tdSql.checkData(0, 8, 0.19) + tdSql.checkData(0, 9, rowNum) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/alter_wal0.py b/tests/pytest/table/alter_wal0.py index 15ad69998f450b8e385cbf58052d246d9de27380..807dd316f2f5b154817217c9cf28821242bc9176 100644 --- a/tests/pytest/table/alter_wal0.py +++ b/tests/pytest/table/alter_wal0.py @@ -43,7 +43,8 @@ class TDTestCase: print("==============step2") tdDnodes.stopAll() - filename = '/var/lib/taos/mnode/wal/wal0' + path = tdDnodes.getDnodesRootDir() + filename = path + '/dnode1/data/mnode/wal/wal0' with open(filename, 'rb') as f1: temp = f1.read() @@ -57,12 +58,12 @@ class TDTestCase: print("==============step3") tdSql.execute("use demo;") tdSql.query('show tables;') - tdSql.checkRows(10) + tdSql.checkRows(9) for i in range(11,21): tdSql.execute("CREATE table if not exists test{num} using meters tags({num});".format(num=i)) tdSql.query('show tables;') - tdSql.checkRows(20) + tdSql.checkRows(19) print("==============check table numbers and create 10 tables") diff --git a/tests/pytest/table/create.py b/tests/pytest/table/create.py index 8fedd4e92046f58a94db66a1e696edfe5d253982..ecd4d011418ed7eca0d028262159c5614c0f490c 100644 --- a/tests/pytest/table/create.py +++ b/tests/pytest/table/create.py @@ -39,6 +39,29 @@ class TDTestCase: except Exception as e: tdLog.exit(e) + # case for defect: https://jira.taosdata.com:18080/browse/TD-2560 + tdSql.execute("create table db.tb02 using st tags(2)") + tdSql.execute("create table db.tb03 using st tags(3)") + tdSql.execute("create table db.tb04 using st tags(4)") + + tdSql.query("show tables like 'tb%' ") + tdSql.checkRows(4) + + tdSql.query("show tables like 'tb0%' ") + tdSql.checkRows(3) + + tdSql.execute("create table db.st0 (ts timestamp, i int) tags(j int)") + tdSql.execute("create table db.st1 (ts timestamp, i int, c2 int) tags(j int, loc nchar(20))") + + tdSql.query("show stables like 'st%' ") + tdSql.checkRows(3) + + # case for defect: https://jira.taosdata.com:18080/browse/TD-2693 + tdSql.execute("create database db2") + tdSql.execute("use db2") + tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") + tdSql.error("insert into db2.tb6 using db2.stb tags(1) values(now 1) tb2 using db2. tags( )values(now 2)") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/table/createTableFromAnotherDb.py b/tests/pytest/table/createTableFromAnotherDb.py new file mode 100644 index 0000000000000000000000000000000000000000..b40e72404cd0eb2e887199eca9964a0765106124 --- /dev/null +++ b/tests/pytest/table/createTableFromAnotherDb.py @@ -0,0 +1,41 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table db.cars(ts timestamp, c int) tags(id int);") + tdSql.execute("create database db2") + tdSql.error("create table db2.car1 using db.cars tags(1)") + tdSql.error("insert into db2.car1 using db1.cars tags(1) values(now, 1);") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/max_table_length.py b/tests/pytest/table/max_table_length.py new file mode 100644 index 0000000000000000000000000000000000000000..ec34f3008fccdb200e645183a5cfdf577fc27503 --- /dev/null +++ b/tests/pytest/table/max_table_length.py @@ -0,0 +1,55 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, db_test.stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + + tdLog.info("check nchar") + tdSql.error("create database anal (ts timestamp ,i nchar(4094))") + tdSql.execute( + "create table anal (ts timestamp ,i nchar(4093))") + + print("==============step2") + tdLog.info("check binary") + tdSql.error("create database anal (ts timestamp ,i binary(16375))") + tdSql.execute( + "create table anal1 (ts timestamp ,i binary(16374))") + + print("==============step3") + tdLog.info("check int & binary") + tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)") + tdSql.execute("create table anal2 (ts timestamp ,i binary(16370),j int)") + tdSql.execute("create table anal3 (ts timestamp ,i binary(16366), j int, k int)") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/add.py b/tests/pytest/tag_lite/add.py index 49e2f8ab6990c24e90e9d1b01df957bc0d3107cd..eb52e9cb9b8f595d77e4d816c2fb2c1317501cfb 100644 --- a/tests/pytest/tag_lite/add.py +++ b/tests/pytest/tag_lite/add.py @@ -141,6 +141,98 @@ class TDTestCase: # TSIM: sql select * from $mt where tgcol2 = 1 -x step2 tdLog.info('select * from %s where tgcol2 = 1 -x step2' % (mt)) tdSql.error('select * from %s where tgcol2 = 1' % (mt)) + tdLog.info('=============== step2-1') + # TSIM: $i = 2 + i = 21 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # bool, tgcol2 int) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bool, tgcol2 int unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bool, tgcol2 int unsigned)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( 1, 2 ) + tdLog.info('create table %s using %s tags( 1, 2 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1, 2 )' % (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol2 = 2 + tdLog.info('select * from %s where tgcol2 = 2' % (mt)) + tdSql.query('select * from %s where tgcol2 = 2' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2 then + tdLog.info('tdSql.checkData(0, 3, 2)') + tdSql.checkData(0, 3, 2) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol2 + tdLog.info('alter table %s drop tag tgcol2' % (mt)) + tdSql.execute('alter table %s drop tag tgcol2' % (mt)) + # TSIM: sql alter table $mt add tag tgcol4 int + tdLog.info('alter table %s add tag tgcol4 int unsigned' % (mt)) + tdSql.execute('alter table %s add tag tgcol4 int unsigned' % (mt)) + tdLog.info('select * from %s where tgcol4=6' % (mt)) + tdSql.query('select * from %s where tgcol4=6' % (mt)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: sql alter table $tb set tag tgcol4 =4 + tdLog.info('alter table %s set tag tgcol4 =4' % (tb)) + tdSql.execute('alter table %s set tag tgcol4 =4' % (tb)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: + # TSIM: sql select * from $mt where tgcol4 = 4 + tdLog.info('select * from %s where tgcol4 = 4' % (mt)) + tdSql.query('select * from %s where tgcol4 = 4' % (mt)) + # TSIM: print $data01 $data02 $data03 + tdLog.info('$data01 $data02 $data03') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 4 then + tdLog.info('tdSql.checkData(0, 3, 4)') + tdSql.checkData(0, 3, 4) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where tgcol2 = 1 -x step2 + tdLog.info('select * from %s where tgcol2 = 1 -x step2' % (mt)) + tdSql.error('select * from %s where tgcol2 = 1' % (mt)) # TSIM: return -1 # TSIM: step2: # TSIM: @@ -235,6 +327,96 @@ class TDTestCase: # TSIM: sql select * from $mt where tgcol2 = 1 -x step3 tdLog.info('select * from %s where tgcol2 = 1 -x step3' % (mt)) tdSql.error('select * from %s where tgcol2 = 1' % (mt)) + tdLog.info('=============== step3-1') + # TSIM: $i = 3 + i = 31 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # smallint, tgcol2 tinyint) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 smallint unsigned, tgcol2 tinyint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 smallint unsigned, tgcol2 tinyint unsigned)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( 1, 2 ) + tdLog.info('create table %s using %s tags( 1, 2 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1, 2 )' % (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol2 = 2 + tdLog.info('select * from %s where tgcol2 = 2' % (mt)) + tdSql.query('select * from %s where tgcol2 = 2' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2 then + tdLog.info('tdSql.checkData(0, 3, 2)') + tdSql.checkData(0, 3, 2) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol2 + tdLog.info('alter table %s drop tag tgcol2' % (mt)) + tdSql.execute('alter table %s drop tag tgcol2' % (mt)) + # TSIM: sql alter table $mt add tag tgcol4 tinyint + tdLog.info('alter table %s add tag tgcol4 tinyint unsigned' % (mt)) + tdSql.execute('alter table %s add tag tgcol4 tinyint unsigned' % (mt)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: sql alter table $tb set tag tgcol4=4 + tdLog.info('alter table %s set tag tgcol4=4' % (tb)) + tdSql.execute('alter table %s set tag tgcol4=4' % (tb)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: + # TSIM: sql select * from $mt where tgcol4 = 4 + tdLog.info('select * from %s where tgcol4 = 4' % (mt)) + tdSql.query('select * from %s where tgcol4 = 4' % (mt)) + # TSIM: print $data01 $data02 $data03 + tdLog.info('$data01 $data02 $data03') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 4 then + tdLog.info('tdSql.checkData(0, 3, 4)') + tdSql.checkData(0, 3, 4) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where tgcol2 = 1 -x step3 + tdLog.info('select * from %s where tgcol2 = 1 -x step3' % (mt)) + tdSql.error('select * from %s where tgcol2 = 1' % (mt)) # TSIM: return -1 # TSIM: step3: # TSIM: @@ -353,6 +535,120 @@ class TDTestCase: # TSIM: sql select * from $mt where tgcol2 = 1 -x step4 tdLog.info('select * from %s where tgcol2 = 1 -x step4' % (mt)) tdSql.error('select * from %s where tgcol2 = 1' % (mt)) + tdLog.info('=============== step4-1') + # TSIM: $i = 4 + i = 41 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # bigint, tgcol2 float) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bigint unsigned, tgcol2 float)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bigint unsigned, tgcol2 float)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( 1, 2 ) + tdLog.info('create table %s using %s tags( 1, 2 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1, 2 )' % (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol2 = 2 + tdLog.info('select * from %s where tgcol2 = 2' % (mt)) + tdSql.query('select * from %s where tgcol2 = 2' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2.00000 then + tdLog.info('tdSql.checkData(0, 3, 2.00000)') + tdSql.checkData(0, 3, 2.00000) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql describe $tb + tdLog.info('describe %s' % (tb)) + tdSql.query('describe %s' % (tb)) + # TSIM: if $data21 != BIGINT then + tdLog.info('tdSql.checkDataType(2, 1, "BIGINT UNSIGNED")') + tdSql.checkDataType(2, 1, "BIGINT") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data31 != FLOAT then + tdLog.info('tdSql.checkDataType(3, 1, "FLOAT")') + tdSql.checkDataType(3, 1, "FLOAT") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data23 != 1 then + tdLog.info('tdSql.checkData(2, 3, TAG)') + tdSql.checkData(2, 3, "TAG") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data33 != 2.000000 then + tdLog.info('tdSql.checkData(3, 3, 2.000000)') + tdSql.checkData(3, 3, "TAG") + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol2 + tdLog.info('alter table %s drop tag tgcol2' % (mt)) + tdSql.execute('alter table %s drop tag tgcol2' % (mt)) + # TSIM: sql alter table $mt add tag tgcol4 float + tdLog.info('alter table %s add tag tgcol4 float' % (mt)) + tdSql.execute('alter table %s add tag tgcol4 float' % (mt)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: sql alter table $tb set tag tgcol4=4 + tdLog.info('alter table %s set tag tgcol4=4' % (tb)) + tdSql.execute('alter table %s set tag tgcol4=4' % (tb)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: + # TSIM: sql select * from $mt where tgcol4 = 4 + tdLog.info('select * from %s where tgcol4 = 4' % (mt)) + tdSql.query('select * from %s where tgcol4 = 4' % (mt)) + # TSIM: print $data01 $data02 $data03 + tdLog.info('$data01 $data02 $data03') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 4.00000 then + tdLog.info('tdSql.checkData(0, 3, 4.00000)') + tdSql.checkData(0, 3, 4.00000) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where tgcol2 = 1 -x step4 + tdLog.info('select * from %s where tgcol2 = 1 -x step4' % (mt)) + tdSql.error('select * from %s where tgcol2 = 1' % (mt)) # TSIM: return -1 # TSIM: step4: # TSIM: @@ -408,12 +704,18 @@ class TDTestCase: # TSIM: sql alter table $mt add tag tgcol4 smallint tdLog.info('alter table %s add tag tgcol4 smallint' % (mt)) tdSql.execute('alter table %s add tag tgcol4 smallint' % (mt)) + # TSIM: sql alter table $mt add tag tgcol5 smallint unsigned + tdLog.info('alter table %s add tag tgcol5 smallint unsigned' % (mt)) + tdSql.execute('alter table %s add tag tgcol5 smallint unsigned' % (mt)) # TSIM: sql reset query cache tdLog.info('reset query cache') tdSql.execute('reset query cache') # TSIM: sql alter table $tb set tag tgcol4=4 tdLog.info('alter table %s set tag tgcol4=4' % (tb)) tdSql.execute('alter table %s set tag tgcol4=4' % (tb)) + # TSIM: sql alter table $tb set tag tgcol5=5 + tdLog.info('alter table %s set tag tgcol5=5' % (tb)) + tdSql.execute('alter table %s set tag tgcol5=5' % (tb)) # TSIM: sql reset query cache tdLog.info('reset query cache') tdSql.execute('reset query cache') @@ -441,6 +743,11 @@ class TDTestCase: # TSIM: if $data03 != 4 then tdLog.info('tdSql.checkData(0, 3, 4)') tdSql.checkData(0, 3, 4) + tdSql.query('select * from %s where tgcol5 = 5' % (mt)) + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1.000000000) + tdSql.checkData(0, 4, 5) # TSIM: return -1 # TSIM: endi # TSIM: diff --git a/tests/pytest/tag_lite/alter_tag.py b/tests/pytest/tag_lite/alter_tag.py new file mode 100644 index 0000000000000000000000000000000000000000..9e5abb6c134840ecb4ab52c7d3a6ab623885e12b --- /dev/null +++ b/tests/pytest/tag_lite/alter_tag.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "CREATE TABLE IF NOT EXISTS ampere (ts TIMESTAMP(8),ampere DOUBLE(8)) TAGS (device_name BINARY(50),build_id BINARY(50),project_id BINARY(50),alias BINARY(50))") + tdSql.execute("insert into d1001 using ampere tags('test', '2', '2', '2') VALUES (now, 123)") + tdSql.execute("ALTER TABLE ampere ADD TAG variable_id BINARY(50)") + + print("==============step2") + + tdSql.execute("insert into d1002 using ampere tags('test', '2', '2', '2', 'test') VALUES (now, 124)") + + tdSql.query("select * from ampere") + tdSql.checkRows(2) + tdSql.checkData(0, 6, None) + tdSql.checkData(1, 6, 'test') + + # Test case for: https://jira.taosdata.com:18080/browse/TD-2423 + tdSql.execute("create table stb(ts timestamp, col1 int, col2 nchar(20)) tags(tg1 int, tg2 binary(20), tg3 nchar(25))") + tdSql.execute("insert into tb1 using stb(tg1, tg3) tags(1, 'test1') values(now, 1, 'test1')") + tdSql.query("select *, tg1, tg2, tg3 from tb1") + tdSql.checkRows(1) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, None) + tdSql.checkData(0, 5, 'test1') + + tdSql.execute("create table tb2 using stb(tg3, tg2) tags('test3', 'test2')") + tdSql.query("select tg1, tg2, tg3 from tb2") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, 'test2') + tdSql.checkData(0, 2, 'test3') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/delete.py b/tests/pytest/tag_lite/delete.py index 17b4deb815e7a05aaa5eaf33685b95e144d3d59b..5e82a96bf41608782fca09d6aeb8b18aa1a727ef 100644 --- a/tests/pytest/tag_lite/delete.py +++ b/tests/pytest/tag_lite/delete.py @@ -869,6 +869,91 @@ class TDTestCase: # TSIM: sql alter table $mt drop tag tgcol6 tdLog.info('alter table %s drop tag tgcol6' % (mt)) tdSql.execute('alter table %s drop tag tgcol6' % (mt)) + tdLog.info('=============== step14') + # TSIM: $i = 14 + i = 14 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # binary(10), tgcol2 int, tgcol3 smallint, tgcol4 binary(11), tgcol5 + # double, tgcol6 binary(20)) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 binary(10), tgcol2 int unsigned, tgcol3 smallint unsigned, tgcol4 binary(11), tgcol5 double, tgcol6 binary(20), tgcol7 tinyint unsigned, tgcol8 bigint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 binary(10), tgcol2 int unsigned, tgcol3 smallint unsigned, tgcol4 binary(11), tgcol5 double, tgcol6 binary(20), tgcol7 tinyint unsigned, tgcol8 bigint unsigned)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( '1', 2, 3, '4', 5, '6' ) + tdLog.info( + 'create table %s using %s tags( "1", 2, 3, "4", 5, "6", 7, 8 )' % + (tb, mt)) + tdSql.execute( + 'create table %s using %s tags( "1", 2, 3, "4", 5, "6", 7, 8 )' % + (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol1 = '1' + tdLog.info('select * from %s where tgcol1 = "1"' % (mt)) + tdSql.query('select * from %s where tgcol1 = "1"' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, "1")') + tdSql.checkData(0, 2, "1") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2 then + tdLog.info('tdSql.checkData(0, 3, 2)') + tdSql.checkData(0, 3, 2) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data04 != 3 then + tdLog.info('tdSql.checkData(0, 4, 3)') + tdSql.checkData(0, 4, 3) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data05 != 4 then + tdLog.info('tdSql.checkData(0, 5, "4")') + tdSql.checkData(0, 5, "4") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data06 != 5.000000000 then + tdLog.info('tdSql.checkData(0, 6, 5.000000000)') + tdSql.checkData(0, 6, 5.000000000) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data07 != 6 then + tdLog.info('tdSql.checkData(0, 7, "6")') + tdSql.checkData(0, 7, "6") + tdLog.info('tdSql.checkData(0, 8, 7)') + tdSql.checkData(0, 8, 7) + tdLog.info('tdSql.checkData(0, 9, 8)') + tdSql.checkData(0, 9, 8) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol3 + tdLog.info('alter table %s drop tag tgcol3' % (mt)) + tdSql.execute('alter table %s drop tag tgcol3' % (mt)) + # TSIM: sql alter table $mt drop tag tgcol4 + tdLog.info('alter table %s drop tag tgcol4' % (mt)) + tdSql.execute('alter table %s drop tag tgcol4' % (mt)) + # TSIM: sql alter table $mt drop tag tgcol6 + tdLog.info('alter table %s drop tag tgcol6' % (mt)) + tdSql.execute('alter table %s drop tag tgcol6' % (mt)) + tdLog.info('alter table %s drop tag tgcol8' % (mt)) + tdSql.execute('alter table %s drop tag tgcol8' % (mt)) # TSIM: # TSIM: sleep 5000 # TSIM: @@ -1556,7 +1641,7 @@ class TDTestCase: # TSIM: print =============== step14 tdLog.info('=============== step14') # TSIM: $i = 14 - i = 14 + i = 20 # TSIM: $mt = $mtPrefix . $i mt = "%s%d" % (mtPrefix, i) # TSIM: $tb = $tbPrefix . $i diff --git a/tests/pytest/tag_lite/unsignedBigint.py b/tests/pytest/tag_lite/unsignedBigint.py new file mode 100644 index 0000000000000000000000000000000000000000..032dacd366437c378b0b8c9b553d6495966cd563 --- /dev/null +++ b/tests/pytest/tag_lite/unsignedBigint.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # bigint unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol bigint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol bigint unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev bigint unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 64) - 1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 64) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/unsignedInt.py b/tests/pytest/tag_lite/unsignedInt.py new file mode 100644 index 0000000000000000000000000000000000000000..6efe12edf110f70d9c2c0187429a0696372bae49 --- /dev/null +++ b/tests/pytest/tag_lite/unsignedInt.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # int unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol int unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol int unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev int unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 32) - 1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 32) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/unsignedSmallint.py b/tests/pytest/tag_lite/unsignedSmallint.py new file mode 100644 index 0000000000000000000000000000000000000000..59cb33dc18eb74b27e9a916e921e10cbabf1b4eb --- /dev/null +++ b/tests/pytest/tag_lite/unsignedSmallint.py @@ -0,0 +1,603 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # smallint unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol smallint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol smallint unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev smallint unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 16)-1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 16) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/unsignedTinyint.py b/tests/pytest/tag_lite/unsignedTinyint.py new file mode 100644 index 0000000000000000000000000000000000000000..1250b08d1b8dd4d3cb5f61af424576146ab70967 --- /dev/null +++ b/tests/pytest/tag_lite/unsignedTinyint.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # tinyint unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol tinyint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol tinyint unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev tinyint unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 8) - 1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 8) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index a2c35444b55fea9df55e46b4c81a9150b642b452..c7781f20873066b02416049e8c472e07c0c3e328 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -111,13 +111,28 @@ if __name__ == "__main__": tdLog.info('stop All dnodes') sys.exit(0) - + tdDnodes.init(deployPath) tdDnodes.setTestCluster(testCluster) tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() - tdDnodes.deploy(1) + is_test_framework = 0 + key_word = 'tdCases.addLinux' + try: + if key_word in open(fileName).read(): + is_test_framework = 1 + except: + pass + if is_test_framework: + moduleName = fileName.replace(".py", "").replace("/", ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + tdDnodes.deploy(1,{}) tdDnodes.start(1) if masterIp == "": diff --git a/tests/pytest/tools/lowaTest.py b/tests/pytest/tools/lowaTest.py index 523229dd463d54c5b2cd23a9a3d4d547858a3b5c..ad8b5925bd99b9c5918421eb277cea6e5ed100a7 100644 --- a/tests/pytest/tools/lowaTest.py +++ b/tests/pytest/tools/lowaTest.py @@ -51,7 +51,7 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - os.system("yes | %slowa -f tools/insert.json" % binPath) + os.system("yes | %staosdemo -f tools/insert.json" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py new file mode 100644 index 0000000000000000000000000000000000000000..28f451b6a08a43508352c75b0c498251c9afca54 --- /dev/null +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -0,0 +1,119 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +import datetime +import csv +import random +import pandas as pd +import argparse +import os.path + +class taosdemoPerformace: + def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay): + self.commitID = commitID + self.dbName = dbName + self.createTableTime = createTableTime + self.insertRecordsTime = insertRecordsTime + self.recordsPerSecond = recordsPerSecond + self.avgDelay = avgDelay + self.maxDelay = maxDelay + self.minDelay = minDelay + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taosperf" + self.conn = taos.connect( + self.host, + self.user, + self.password, + self.config) + + def createTablesAndStoreData(self): + cursor = self.conn.cursor() + + cursor.execute("create database if not exists %s" % self.dbName) + cursor.execute("use %s" % self.dbName) + cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") + print("==================== taosdemo performance ====================") + print("create tables time: %f" % self.createTableTime) + print("insert records time: %f" % self.insertRecordsTime) + print("records per second: %f" % self.recordsPerSecond) + print("avg delay: %f" % self.avgDelay) + print("max delay: %f" % self.maxDelay) + print("min delay: %f" % self.minDelay) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) + cursor.execute("drop database if exists taosdemo_insert_test") + + cursor.close() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-c', + '--commit-id', + action='store', + type=str, + help='git commit id (default: null)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='perf', + type=str, + help='Database name to be created (default: perf)') + parser.add_argument( + '-t', + '--create-table', + action='store', + type=float, + help='create table time') + parser.add_argument( + '-i', + '--insert-records', + action='store', + type=float, + help='insert records time') + parser.add_argument( + '-r', + '---records-per-second', + action='store', + type=float, + help='records per request') + parser.add_argument( + '-avg', + '---avg-delay', + action='store', + type=float, + help='avg delay') + parser.add_argument( + '-max', + '---max-delay', + action='store', + type=float, + help='max delay') + parser.add_argument( + '-min', + '---min-delay', + action='store', + type=float, + help='min delay') + + + args = parser.parse_args() + + perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, + args.avg_delay, args.max_delay, args.min_delay) + perftest.createTablesAndStoreData() \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 1a976aef594576ba29b928465fc38baea6dcb63e..c450570d2446a236ad8fa98c0817d2e31bcae795 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -23,9 +23,10 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - - self.numberOfTables = 10000 + + self.numberOfTables = 1000 self.numberOfRecords = 100 + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -38,9 +39,10 @@ class TDTestCase: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + buildPath = root[:len(root) - len("/build/bin")] break return buildPath + def run(self): tdSql.prepare() buildPath = self.getBuildPath() @@ -48,18 +50,23 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -t %d -n %d" % (binPath,self.numberOfTables, self.numberOfRecords)) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -y -M -t %d -n %d -x" % + (binPath, self.numberOfTables, self.numberOfRecords)) tdSql.execute("use test") tdSql.query("select count(*) from meters") tdSql.checkData(0, 0, self.numberOfTables * self.numberOfRecords) - tdSql.query("select sum(f1) from test.meters interval(1h) sliding(30m)") + tdSql.query( + "select sum(col1) from test.meters interval(1h) sliding(30m)") tdSql.checkRows(2) - tdSql.query("select apercentile(f1, 1) from test.meters interval(10s)") - tdSql.checkRows(11) + tdSql.query( + "select apercentile(col1, 1) from test.meters interval(10s)") + tdSql.checkRows(1) + + tdSql.error("select loc, count(loc) from test.meters") def stop(self): tdSql.close() diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTest2.py new file mode 100644 index 0000000000000000000000000000000000000000..75a79d0585e766718151ca9b0e3e195c03732e16 --- /dev/null +++ b/tests/pytest/tools/taosdemoTest2.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import threading +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numberOfTables = 10 + self.numberOfRecords = 1000000 + + def insertDataAndAlterTable(self, threadID): + if(threadID == 0): + os.system("taosdemo -M -y -t %d -n %d -x" % + (self.numberOfTables, self.numberOfRecords)) + if(threadID == 1): + time.sleep(2) + print("use test") + tdSql.execute("use test") + # check if all the tables have heen created + while True: + tdSql.query("show tables") + rows = tdSql.queryRows + print("number of tables: %d" % rows) + if(rows == self.numberOfTables): + break + time.sleep(1) + # check if there are any records in the last created table + while True: + print("query started") + tdSql.query("select * from test.t9") + rows = tdSql.queryRows + print("number of records: %d" % rows) + if(rows > 0): + break + time.sleep(1) + print("alter table test.meters add column col10 int") + tdSql.execute("alter table test.meters add column col10 int") + print("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + + def run(self): + tdSql.prepare() + + t1 = threading.Thread(target=self.insertDataAndAlterTable, args=(0, )) + t2 = threading.Thread(target=self.insertDataAndAlterTable, args=(1, )) + + t1.start() + t2.start() + t1.join() + t2.join() + + tdSql.query("select count(*) from test.meters") + tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index 3169b748e0843c720beb54eced28a14f1ca747a6..867ee696a261936e0118250995da4da5dca7ffb5 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -38,38 +38,62 @@ class TDTestCase: insertRows = 200 t0 = 1604298064000 + sql='insert into db.t1 values ' + temp='' tdLog.info("insert %d rows" % (insertRows)) for i in range(0, insertRows): - ret = tdSql.execute( - 'insert into t1 values (%d , 1)' % - (t0+i)) + # ret = tdSql.execute( + # 'insert into t1 values (%d , 1)' % + # (t0+i)) + temp += '(%d,1)' %(t0+i) + if i % 100 == 0 or i == (insertRows - 1 ): + print(sql+temp) + ret = tdSql.execute( + sql+temp + ) + temp = '' print("==========step2") print("restart to commit ") tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows) + for k in range(0,100): tdLog.info("insert %d rows" % (insertRows)) + temp='' for i in range (0,insertRows): - ret = tdSql.execute( - 'insert into db.t1 values(%d,1)' % - (t0+k*200+i) - ) + temp += '(%d,1)' %(t0+k*200+i) + if i % 100 == 0 or i == (insertRows - 1 ): + print(sql+temp) + ret = tdSql.execute( + sql+temp + ) + temp = '' + tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows+200*k) - print("==========step2") + print("==========step3") print("insert into another table ") s = 'use db' tdSql.execute(s) ret = tdSql.execute('create table t2 (ts timestamp, a int)') insertRows = 20000 + sql = 'insert into t2 values ' + temp = '' for i in range(0, insertRows): - ret = tdSql.execute( - 'insert into t2 values (%d, 1)' % - (t0+i)) + # ret = tdSql.execute( + # 'insert into t2 values (%d, 1)' % + # (t0+i)) + temp += '(%d,1)' %(t0+i) + if i % 500 == 0 or i == (insertRows - 1 ): + print(sql+temp) + ret = tdSql.execute( + sql+temp + ) + temp = '' tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t2") diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 757399b4a262dff7b11619791d3c82686fb293e8..91c75efe9762a18042660423708e6dd9475166dd 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -108,6 +108,36 @@ class TDDnode: self.deployed = 0 self.testCluster = False self.valgrind = 0 + self.cfgDict = { + "numOfLogLines":"100000000", + "mnodeEqualVnodeNum":"0", + "walLevel":"2", + "fsync":"1000", + "statusInterval":"1", + "numOfMnodes":"3", + "numOfThreadsPerCore":"2.0", + "monitor":"0", + "maxVnodeConnections":"30000", + "maxMgmtConnections":"30000", + "maxMeterConnections":"30000", + "maxShellConns":"30000", + "locale":"en_US.UTF-8", + "charset":"UTF-8", + "asyncLog":"0", + "anyIp":"0", + "tsEnableTelemetryReporting":"0", + "dDebugFlag":"135", + "mDebugFlag":"135", + "sdbDebugFlag":"135", + "rpcDebugFlag":"135", + "tmrDebugFlag":"131", + "cDebugFlag":"135", + "httpDebugFlag":"135", + "monitorDebugFlag":"135", + "udebugFlag":"135", + "jnidebugFlag":"135", + "qdebugFlag":"135" + } def init(self, path): self.path = path @@ -131,7 +161,10 @@ class TDDnode: return totalSize - def deploy(self): + def addExtraCfg(self, option, value): + self.cfgDict.update({option: value}) + + def deploy(self, *updatecfgDict): self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) @@ -175,36 +208,27 @@ class TDDnode: self.cfg("publicIp", "192.168.0.%d" % (self.index)) self.cfg("internalIp", "192.168.0.%d" % (self.index)) self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("tsEnableTelemetryReporting", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") + self.cfgDict["dataDir"] = self.dataDir + self.cfgDict["logDir"] = self.logDir + # self.cfg("dataDir",self.dataDir) + # self.cfg("logDir",self.logDir) + # print(updatecfgDict) + isFirstDir = 1 + if updatecfgDict[0] and updatecfgDict[0][0]: + print(updatecfgDict[0][0]) + for key,value in updatecfgDict[0][0].items(): + if value == 'dataDir' : + if isFirstDir: + self.cfgDict.pop('dataDir') + self.cfg(value,key) + isFirstDir = 0 + else: + self.cfg(value,key) + else: + self.addExtraCfg(key,value) + for key, value in self.cfgDict.items(): + self.cfg(key, value) + self.deployed = 1 tdLog.debug( "dnode:%d is deployed and configured by %s" % @@ -255,9 +279,35 @@ class TDDnode: tdLog.exit(cmd) self.running = 1 tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + if self.valgrind == 0: + time.sleep(0.1) + key = 'from offline to online' + bkey = bytes(key,encoding="utf8") + logFile = self.logDir + "/taosdlog.0" + i = 0 + while not os.path.exists(logFile): + sleep(0.1) + i += 1 + if i>50: + break + popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + pid = popen.pid + # print('Popen.pid:' + str(pid)) + timeout = time.time() + 60*2 + while True: + line = popen.stdout.readline().strip() + if bkey in line: + popen.kill() + break + if time.time() > timeout: + tdLog.exit('wait too long for taosd start') + tdLog.debug("the dnode:%d has been started." % (self.index)) + else: + tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index)) + time.sleep(10) - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) + + # time.sleep(5) def startWithoutSleep(self): buildPath = self.getBuildPath() @@ -437,7 +487,7 @@ class TDDnodes: def setValgrind(self, value): self.valgrind = value - def deploy(self, index): + def deploy(self, index, *updatecfgDict): self.sim.setTestCluster(self.testCluster) if (self.simDeployed == False): @@ -447,7 +497,7 @@ class TDDnodes: self.check(index) self.dnodes[index - 1].setTestCluster(self.testCluster) self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() + self.dnodes[index - 1].deploy(updatecfgDict) def cfg(self, index, option, value): self.check(index) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b2ed6212fd643c158f7ed6f4cc6cb2449a512a2e..16931cca333a3300b7d0d6831bbb51db0238b1d1 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -16,9 +16,12 @@ import os import time import datetime import inspect +import psutil +import shutil from util.log import * + class TDSql: def __init__(self): self.queryRows = 0 @@ -183,6 +186,55 @@ class TDSql: tdLog.exit("%s(%d) failed: sql:%s, affectedRows:%d != expect:%d" % args) tdLog.info("sql:%s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) - - + + def taosdStatus(self, state): + tdLog.sleep(5) + pstate = 0 + for i in range(30): + pstate = 0 + pl = psutil.pids() + for pid in pl: + try: + if psutil.Process(pid).name() == 'taosd': + print('have already started') + pstate = 1 + break + except psutil.NoSuchProcess: + pass + if pstate == state :break + if state or pstate: + tdLog.sleep(1) + continue + pstate = 0 + break + + args=(pstate,state) + if pstate == state: + tdLog.info("taosd state is %d == expect:%d" %args) + else: + tdLog.exit("taosd state is %d != expect:%d" %args) + pass + + def haveFile(self, dir, state): + if os.path.exists(dir) and os.path.isdir(dir): + if not os.listdir(dir): + if state : + tdLog.exit("dir: %s is empty, expect: not empty" %dir) + else: + tdLog.info("dir: %s is empty, expect: empty" %dir) + else: + if state : + tdLog.info("dir: %s is not empty, expect: not empty" %dir) + else: + tdLog.exit("dir: %s is not empty, expect: empty" %dir) + else: + tdLog.exit("dir: %s doesn't exist" %dir) + def createDir(self, dir): + if os.path.exists(dir): + shutil.rmtree(dir) + tdLog.info("dir: %s is removed" %dir) + os.makedirs( dir, 755 ) + tdLog.info("dir: %s is created" %dir) + pass + tdSql = TDSql() diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 4dd3957f390237c6dddc6b1f66b14ee177b6dbcf..cde51ebdbfc90f50bbe8d7941fd4f697d8b59ee5 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -134,8 +134,6 @@ run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim #unsupport run general/parser/slimit_alter_tags.sim -#unsupport run general/parser/stream_on_sys.sim -#unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim run general/stable/metrics.sim @@ -213,9 +211,6 @@ run general/vector/table_time.sim run general/stream/restart_stream.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/general/alter/cached_schema_after_alter.sim b/tests/script/general/alter/cached_schema_after_alter.sim index 2d049ec5955e281d866eaaa15abf06583909c87d..96ee4390845450d53508cc90c48a3148a0a827dd 100644 --- a/tests/script/general/alter/cached_schema_after_alter.sim +++ b/tests/script/general/alter/cached_schema_after_alter.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $db = csaa_db @@ -53,10 +53,10 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -sleep 5000 +sleep 3000 sql connect sql use $db diff --git a/tests/script/general/alter/count.sim b/tests/script/general/alter/count.sim index 157b4c137198b70a9e7ad3b1f64e5b4b06a90415..fc936668b8ea08f9cd08874ad98668a4d8904315 100644 --- a/tests/script/general/alter/count.sim +++ b/tests/script/general/alter/count.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 print ========= start dnode1 as master system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -141,9 +141,9 @@ endi print ============= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb if $data00 != 24 then @@ -250,9 +250,9 @@ endi print ============== step18 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 #sql select count(g) from tb #if $data00 != 12 then diff --git a/tests/script/general/alter/dnode.sim b/tests/script/general/alter/dnode.sim index 73a095ec054320d0373f137250193bc4d8230234..7b31218fc231cfdbb79ca97573cfc6f6f149037d 100644 --- a/tests/script/general/alter/dnode.sim +++ b/tests/script/general/alter/dnode.sim @@ -4,14 +4,14 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 sql alter dnode 1 resetlog sql alter dnode 1 monitor 1 -sleep 5000 +sleep 3000 sql select * from log.dn if $rows <= 0 then return -1 diff --git a/tests/script/general/alter/import.sim b/tests/script/general/alter/import.sim index 76388a26f2b15d1ce9a86331ab2e70cb29a88480..aef0a258b24563e915cd8aa3dd42f6623a29170a 100644 --- a/tests/script/general/alter/import.sim +++ b/tests/script/general/alter/import.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 print ========= start dnode1 as master system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/alter/insert1.sim b/tests/script/general/alter/insert1.sim index 3b16214465c15133aecc3e2c5977996cbaa22b5f..12ab09beb989dd963a9e8c9c3ff5926e78d8b0ac 100644 --- a/tests/script/general/alter/insert1.sim +++ b/tests/script/general/alter/insert1.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -940,9 +940,9 @@ endi print ======== step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql select * from tb order by ts asc if $rows != 8 then diff --git a/tests/script/general/alter/insert2.sim b/tests/script/general/alter/insert2.sim index 28948b69d231952f3baae0d79a75c1f2c9c6fb60..dcd9f500304f906ddddb33bd1a04c5943c232d49 100644 --- a/tests/script/general/alter/insert2.sim +++ b/tests/script/general/alter/insert2.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -608,9 +608,9 @@ sql_error alter table tb drop column a print ======== step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql select * from tb order by ts desc if $rows != 7 then diff --git a/tests/script/general/alter/metrics.sim b/tests/script/general/alter/metrics.sim index 5ed8050b96b90d679fcb85ada06c6065b8f0735a..fd0b210cd1b452b2a35ebcd9f74aec98c3817b03 100644 --- a/tests/script/general/alter/metrics.sim +++ b/tests/script/general/alter/metrics.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -367,9 +367,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d2 sql describe tb diff --git a/tests/script/general/alter/table.sim b/tests/script/general/alter/table.sim index 5a72fc925619e1a0ab40289056f6af355b995ca9..06704eeca6b3149b47ddc2ffb90aaab9df934bd8 100644 --- a/tests/script/general/alter/table.sim +++ b/tests/script/general/alter/table.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -318,9 +318,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d1 sql describe tb diff --git a/tests/script/general/cache/new_metrics.sim b/tests/script/general/cache/new_metrics.sim index 0e304d9acb0ec3ceba18993846f9f6d8269ff966..a13dd23bb64c490f52fed7abe946ec7dfec90adb 100644 --- a/tests/script/general/cache/new_metrics.sim +++ b/tests/script/general/cache/new_metrics.sim @@ -3,10 +3,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/cache/restart_metrics.sim b/tests/script/general/cache/restart_metrics.sim index 18c514acbf5a038f998035302cca8bb0fbd95a0b..f5035e1251c11c7c4010160cfac967d9b2617c21 100644 --- a/tests/script/general/cache/restart_metrics.sim +++ b/tests/script/general/cache/restart_metrics.sim @@ -3,10 +3,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -50,10 +49,9 @@ endi print =============== step2 system sh/exec.sh -n dnode1 -s stop -sleep 5000 +sleep 3000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10 system sh/exec.sh -n dnode1 -s start print =============== step3 diff --git a/tests/script/general/cache/restart_table.sim b/tests/script/general/cache/restart_table.sim index c4e6c6f2acc8f0fb0902a6a6adf8d9b91984a712..fcdb2fb593bac7d3eb13b82c7abfe268eafe36dc 100644 --- a/tests/script/general/cache/restart_table.sim +++ b/tests/script/general/cache/restart_table.sim @@ -3,10 +3,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -34,15 +33,14 @@ endi print =============== step2 system sh/exec.sh -n dnode1 -s stop -sleep 5000 +sleep 3000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10 system sh/exec.sh -n dnode1 -s start print =============== step3 print ==> sleep 8 seconds to renew cache -sleep 3000 +sleep 2000 sql reset query cache sleep 18000 diff --git a/tests/script/general/column/commit.sim b/tests/script/general/column/commit.sim index c574db1aa9dd28139251b4aa0433200e5fde230a..e1b98d38147fa4d5721f4e38915f68370378998d 100644 --- a/tests/script/general/column/commit.sim +++ b/tests/script/general/column/commit.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -89,9 +89,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/column/metrics.sim b/tests/script/general/column/metrics.sim index 673b66c9e266a68171161c88d3e2dfadd9d446da..a46ab6560ae4d76c74408639ad6055eb093cb143 100644 --- a/tests/script/general/column/metrics.sim +++ b/tests/script/general/column/metrics.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -157,9 +157,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/column/table.sim b/tests/script/general/column/table.sim index aec0dc3c75031f9c4e167ffae5511ff34e4ec886..7c9302aa083fd03d6b53190ff38927a5f115d688 100644 --- a/tests/script/general/column/table.sim +++ b/tests/script/general/column/table.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -129,9 +129,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print ============== step5 diff --git a/tests/script/general/compress/commitlog.sim b/tests/script/general/compress/commitlog.sim index b5d653fe83e94045c6f5f7ce7f58104504ad5330..e8eab6ed0cd9b5b6c2ea70042bd0f24e92c512a6 100644 --- a/tests/script/general/compress/commitlog.sim +++ b/tests/script/general/compress/commitlog.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c comp -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -87,9 +87,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/compress/compress.sim b/tests/script/general/compress/compress.sim index 6975f879968107bd142a3595869fc120bc1daa02..0df2a0d4cba3c6fb2b28ac8506e498c650c988b7 100644 --- a/tests/script/general/compress/compress.sim +++ b/tests/script/general/compress/compress.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c comp -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -82,9 +82,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/compress/compress2.sim b/tests/script/general/compress/compress2.sim index cf96f572ace4a693b1c5640986510d2f5a4d9d74..007d1ec3394d0cc141bc8ece6ee3968407eb4c20 100644 --- a/tests/script/general/compress/compress2.sim +++ b/tests/script/general/compress/compress2.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c comp -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -82,9 +82,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/compress/uncompress.sim b/tests/script/general/compress/uncompress.sim index 13d288451c8031154f2c62fe4e5f8670fad8517a..9c71c8c1cc642557f929022f0d46a33399e10abf 100644 --- a/tests/script/general/compress/uncompress.sim +++ b/tests/script/general/compress/uncompress.sim @@ -4,7 +4,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c comp -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -81,9 +81,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 -system sh/exec.sh -n dnode1 -s start sleep 3000 +system sh/exec.sh -n dnode1 -s start +sleep 2000 print =============== step5 diff --git a/tests/script/general/compute/avg.sim b/tests/script/general/compute/avg.sim index 52a270bc6d9b1581fb210693f74cce9ecc73b3fa..027cfbe61deb28dc1767d8a571a0def3c300b628 100644 --- a/tests/script/general/compute/avg.sim +++ b/tests/script/general/compute/avg.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_av_db diff --git a/tests/script/general/compute/bottom.sim b/tests/script/general/compute/bottom.sim index 415bd36e2e0417e8f9da2989e21f1a1908e01cfd..5c76d5d171139cdebe8796fffe390e48809c4cb9 100644 --- a/tests/script/general/compute/bottom.sim +++ b/tests/script/general/compute/bottom.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_bo_db diff --git a/tests/script/general/compute/count.sim b/tests/script/general/compute/count.sim index d8aeb18752a3fee18471a3402d259142486ecc1a..fc481088a5774ea318f62a22d11dc5b375f53b7e 100644 --- a/tests/script/general/compute/count.sim +++ b/tests/script/general/compute/count.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_co_db diff --git a/tests/script/general/compute/diff.sim b/tests/script/general/compute/diff.sim index 88c2ecb09df9128499cef1426a53d5b922512164..433a9356096c4fba8f7049db25c201ae55748e42 100644 --- a/tests/script/general/compute/diff.sim +++ b/tests/script/general/compute/diff.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_di_db diff --git a/tests/script/general/compute/diff2.sim b/tests/script/general/compute/diff2.sim index 14675c823f04693c76fcb151169b6dba738b8aaf..7406771ac6769ffd077eab73f682c1d82a5d3d40 100644 --- a/tests/script/general/compute/diff2.sim +++ b/tests/script/general/compute/diff2.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_di_db diff --git a/tests/script/general/compute/first.sim b/tests/script/general/compute/first.sim index 01b82cce62adbc13420289878ea01ed63776bbce..f12ed2b73a78aba273804452a9121a193baa686c 100644 --- a/tests/script/general/compute/first.sim +++ b/tests/script/general/compute/first.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_fi_db diff --git a/tests/script/general/compute/interval.sim b/tests/script/general/compute/interval.sim index 8f9bf2ecb3013afe2bc11d5c851d8effbcf37686..79f34752223f93ac9a3d6446a934a54a16c60f08 100644 --- a/tests/script/general/compute/interval.sim +++ b/tests/script/general/compute/interval.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_in_db diff --git a/tests/script/general/compute/last.sim b/tests/script/general/compute/last.sim index 88a6a268463ef005dd4c611e2b127c3a0d8f3bf0..7b7c45044f1164bbb1510ae898c8a1ee26421b62 100644 --- a/tests/script/general/compute/last.sim +++ b/tests/script/general/compute/last.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_la_db diff --git a/tests/script/general/compute/last_row.sim b/tests/script/general/compute/last_row.sim index 55d97fe53c75b8df34caed98f4bd57bc05a9711f..03fa9bc4af66d8585b3f2052f01092eb5ee7562e 100644 --- a/tests/script/general/compute/last_row.sim +++ b/tests/script/general/compute/last_row.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_la_db diff --git a/tests/script/general/compute/leastsquare.sim b/tests/script/general/compute/leastsquare.sim index 5a28e74bffeed53e3c7202a2dc5d3bf2551ed94b..20e2cc9d1757fafa25a950a44c0813c70f21aedc 100644 --- a/tests/script/general/compute/leastsquare.sim +++ b/tests/script/general/compute/leastsquare.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_le_db diff --git a/tests/script/general/compute/max.sim b/tests/script/general/compute/max.sim index 1029ba78cc595b70048f887c55a2393cb2b46e10..ba87416292458ffdc3e1c9e37723f08b2d57a03b 100644 --- a/tests/script/general/compute/max.sim +++ b/tests/script/general/compute/max.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_ma_db diff --git a/tests/script/general/compute/min.sim b/tests/script/general/compute/min.sim index d6a57f4f20d01a4ab1fc95ec902785363d1f004b..e981f8335f1242ea5258647298d18e56b3dcc164 100644 --- a/tests/script/general/compute/min.sim +++ b/tests/script/general/compute/min.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mi_db diff --git a/tests/script/general/compute/null.sim b/tests/script/general/compute/null.sim index 47f7a3c1b96152e5a30b4712831146187b98abc7..de2a834684e0a555a9a5ea694f074b59a8a21af1 100644 --- a/tests/script/general/compute/null.sim +++ b/tests/script/general/compute/null.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = db diff --git a/tests/script/general/compute/percentile.sim b/tests/script/general/compute/percentile.sim index 0fb88e7a40224404c2889696d6e8ba34655d18ee..9798aa2f5c6e1c1a2271c255fbd6cce81b4dc4e6 100644 --- a/tests/script/general/compute/percentile.sim +++ b/tests/script/general/compute/percentile.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_pe_db diff --git a/tests/script/general/compute/stddev.sim b/tests/script/general/compute/stddev.sim index abe4025e961460803e33c63c7243a1112a1bfab5..2f6ffb097a91854a714a6c9b24ace8d43726c913 100644 --- a/tests/script/general/compute/stddev.sim +++ b/tests/script/general/compute/stddev.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_st_db diff --git a/tests/script/general/compute/sum.sim b/tests/script/general/compute/sum.sim index 9d42e766b61e4c137b7f738dcb8b27d1f5061413..230248a37093e7e163eba6fe9fc5ca63d386d298 100644 --- a/tests/script/general/compute/sum.sim +++ b/tests/script/general/compute/sum.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_su_db diff --git a/tests/script/general/compute/top.sim b/tests/script/general/compute/top.sim index e6bcd4a5cb0cfb928d8e322b80aa10d801fe76e5..a4f482b12759f9586a4660f0622b463b7a46b9da 100644 --- a/tests/script/general/compute/top.sim +++ b/tests/script/general/compute/top.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_to_db diff --git a/tests/script/general/connection/connection.sim b/tests/script/general/connection/connection.sim index abebbacbd94f0a3fc88083f980da77f7b7df8b9b..1af6e1fda6989c8a9c6eed36e579f8325eee0ac6 100644 --- a/tests/script/general/connection/connection.sim +++ b/tests/script/general/connection/connection.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/general/connection/mqtt.sim b/tests/script/general/connection/mqtt.sim index 4b291f91ea5089018207b11b2016b234dcc2a883..c2c50ef17e1264d6f27f64417f1500bc83346012 100644 --- a/tests/script/general/connection/mqtt.sim +++ b/tests/script/general/connection/mqtt.sim @@ -10,7 +10,7 @@ system sh/cfg.sh -n dnode1 -c mqtt -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database mqttdb; sql create table mqttdb.devices(ts timestamp, value double) tags(name binary(32), model binary(32), serial binary(16), param binary(16), unit binary(16)); diff --git a/tests/script/general/connection/sim.tar.gz b/tests/script/general/connection/sim.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..10bc1a6bace1c8b6796a98c53e3aa3c15e0bfd7c Binary files /dev/null and b/tests/script/general/connection/sim.tar.gz differ diff --git a/tests/script/general/connection/test_old_data.sim b/tests/script/general/connection/test_old_data.sim new file mode 100644 index 0000000000000000000000000000000000000000..83df850f0bb93fd216520bbea4065400aa334091 --- /dev/null +++ b/tests/script/general/connection/test_old_data.sim @@ -0,0 +1,32 @@ +system sh/stop_dnodes.sh +system sh/mv_old_data.sh + +print ============== deploy + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step1 + +sql use test +sql select * from m1 + +print $rows points data are retrieved +if $rows != 7 then + return -1 +endi + +print =============== step 2 + +sql select * from t1 + +print $rows points data are retrieved +if $rows != 7 then + return -1 +endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim index 1c3f543ffdb149d67cfaf1527a0e88f46f626913..170ba21c28b7ff4a79a8f2375dc1d36ba91e6120 100644 --- a/tests/script/general/db/alter_option.sim +++ b/tests/script/general/db/alter_option.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= create database @@ -78,23 +78,10 @@ if $data5_db != 1 then return -1 endi -sql alter database db quorum 2 -sql show databases -print quorum $data5_db -if $data5_db != 2 then - return -1 -endi +sql_error alter database db quorum 2 -sql alter database db quorum 3 -sql show databases -print quorum $data5_db -if $data5_db != 3 then - return -1 -endi +sql_error alter database db quorum 3 -sql alter database db quorum 3 -sql alter database db quorum 2 -sql alter database db quorum 1 sql_error alter database db quorum 0 sql_error alter database db quorum 4 sql_error alter database db quorum 5 @@ -222,7 +209,7 @@ sql alter database db wal 1 sql alter database db wal 2 sql alter database db wal 1 sql alter database db wal 2 -sql_error alter database db wal 0 +sql alter database db wal 0 sql_error alter database db wal 3 sql_error alter database db wal 4 sql_error alter database db wal -1 diff --git a/tests/script/general/db/alter_tables_d2.sim b/tests/script/general/db/alter_tables_d2.sim index 9ef39fb55664ad0f984a3a2b4971daa69feec074..f74f98d571e49090e2c25d2371a0e0c268c9a3ee 100644 --- a/tests/script/general/db/alter_tables_d2.sim +++ b/tests/script/general/db/alter_tables_d2.sim @@ -11,11 +11,26 @@ system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start - -sleep 3000 sql connect sql create dnode $hostname2 -sleep 1000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ============================ step1 @@ -80,10 +95,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -110,7 +138,6 @@ if $rows != 40 then return -1 endi - sql insert into db.t100 values(now, 1) sql insert into db.t101 values(now, 1) sql insert into db.t102 values(now, 1) @@ -144,10 +171,24 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -198,7 +239,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -223,10 +264,10 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql reset query cache sleep 1000 @@ -268,10 +309,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -321,7 +375,7 @@ if $rows != 80 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -351,13 +405,26 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache -sleep 1000 +sleep 100 sql show db.tables if $rows != 80 then @@ -401,10 +468,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step1xx: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1xx +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1xx +endi sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) @@ -454,7 +534,7 @@ if $rows != 100 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then diff --git a/tests/script/general/db/alter_tables_v1.sim b/tests/script/general/db/alter_tables_v1.sim index dde5eb6d9ecfbce4467a54abf883cc6b9ffbb696..20c4c7336312b7eb701fe04c6b4158f466e43146 100644 --- a/tests/script/general/db/alter_tables_v1.sim +++ b/tests/script/general/db/alter_tables_v1.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -51,9 +49,22 @@ print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step2 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t10 using db.st tags(0) sql create table db.t11 using db.st tags(1) @@ -91,9 +102,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 30 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t20 using db.st tags(0) sql create table db.t21 using db.st tags(1) @@ -124,7 +148,7 @@ sql insert into db.t29 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t0 if $rows != 1 then @@ -148,9 +172,22 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step7 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step7 +endi sql reset query cache sleep 1000 @@ -190,9 +227,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 40 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t30 using db.st tags(0) sql create table db.t31 using db.st tags(1) @@ -249,12 +299,24 @@ if $rows != 40 then return -1 endi - print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -294,9 +356,22 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 50 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi sql create table db.t40 using db.st tags(0) sql create table db.t41 using db.st tags(1) diff --git a/tests/script/general/db/alter_tables_v4.sim b/tests/script/general/db/alter_tables_v4.sim index 7c9262874d8f07cef92df2585fe60261ea763080..10bb4e108bf7fc6c879e230565dad29599f15549 100644 --- a/tests/script/general/db/alter_tables_v4.sim +++ b/tests/script/general/db/alter_tables_v4.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -70,9 +68,22 @@ endi print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -131,9 +142,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -184,7 +208,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -208,9 +232,9 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql reset query cache sleep 1000 @@ -250,9 +274,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -331,9 +368,22 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -378,9 +428,24 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi + + sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) sql create table db.t402 using db.st tags(2) diff --git a/tests/script/general/db/alter_vgroups.sim b/tests/script/general/db/alter_vgroups.sim index 13928cf03301f1528db3a1fd9e5951b3115a6125..81ffc7d443ce6b21978d338bfeb01f00f89a17b1 100644 --- a/tests/script/general/db/alter_vgroups.sim +++ b/tests/script/general/db/alter_vgroups.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ step1 @@ -70,9 +70,9 @@ endi print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 2 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -132,9 +132,9 @@ print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) diff --git a/tests/script/general/db/backup/keep.sim b/tests/script/general/db/backup/keep.sim index 29771fc97815c6781cfa3e44fbbfa81760859633..943022deba516fc2dbbc64bdf984659da0233cff 100644 --- a/tests/script/general/db/backup/keep.sim +++ b/tests/script/general/db/backup/keep.sim @@ -23,7 +23,7 @@ sql connect print ========= start other dnodes sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 print ======== step1 create db sql create database keepdb replica 1 keep 30 days 7 @@ -50,9 +50,9 @@ endi print ======== step2 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb print ===> rows $rows @@ -112,9 +112,9 @@ endi print ======== step5 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb print ===> rows $rows @@ -153,9 +153,9 @@ endi print ======== step7 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +sleep 2000 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql select * from tb print ===> rows $rows diff --git a/tests/script/general/db/basic.sim b/tests/script/general/db/basic.sim index 1c4939256be82541f40e34d5c89ea4c1f67de288..684ce825fecb64ef580751a8df104fd4e1c7cf24 100644 --- a/tests/script/general/db/basic.sim +++ b/tests/script/general/db/basic.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/db/basic1.sim b/tests/script/general/db/basic1.sim index 302c5ac40992271e9ef757cb9704beb377a75d16..9ec1aabe98ba21af5927faec2e26aef1c3a14aeb 100644 --- a/tests/script/general/db/basic1.sim +++ b/tests/script/general/db/basic1.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/db/basic2.sim b/tests/script/general/db/basic2.sim index afe5ee5d95402fac985b9c5673fb43e35a285f22..acd035bd744eb53b856c32beaf69445eb24e8315 100644 --- a/tests/script/general/db/basic2.sim +++ b/tests/script/general/db/basic2.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/basic3.sim b/tests/script/general/db/basic3.sim index 88f5bf64604d2f1f6bd11040de844e871808c566..fb64476696c7aba65d5f5f87c68857ed32ff24fc 100644 --- a/tests/script/general/db/basic3.sim +++ b/tests/script/general/db/basic3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/basic4.sim b/tests/script/general/db/basic4.sim index a0a9aaa627fd643b2863b14a4cacb93f092104b0..ce6d352d12412c95d0883ca246e2603b36617bb8 100644 --- a/tests/script/general/db/basic4.sim +++ b/tests/script/general/db/basic4.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/basic5.sim b/tests/script/general/db/basic5.sim index 82b9bf9bf4dadf2c0215626d8905a64cc0962fe3..08c9db23323ba7d4257671d6f0a71c3dd133ad03 100644 --- a/tests/script/general/db/basic5.sim +++ b/tests/script/general/db/basic5.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/delete.sim b/tests/script/general/db/delete.sim index e4d40bf5da267985066f61383c6614dee78aaff1..4384044885e7c6a660079551b1190481ace3554e 100644 --- a/tests/script/general/db/delete.sim +++ b/tests/script/general/db/delete.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -44,7 +44,7 @@ endi print ======= step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start $x = 0 diff --git a/tests/script/general/db/delete_reuse1.sim b/tests/script/general/db/delete_reuse1.sim index feeb0152c1d8dc707a327f89ebf95e242b35294b..b18bb285d1c002b6607a0a6574798baf6a59bb3d 100644 --- a/tests/script/general/db/delete_reuse1.sim +++ b/tests/script/general/db/delete_reuse1.sim @@ -22,7 +22,7 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/db/delete_reuse2.sim b/tests/script/general/db/delete_reuse2.sim index 9053e8af011674162174a718c52d157b1e70984a..c82457ec42921cccf90d3351a09de41d8bf7197c 100644 --- a/tests/script/general/db/delete_reuse2.sim +++ b/tests/script/general/db/delete_reuse2.sim @@ -21,9 +21,9 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes -sleep 2000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 + sql connect sql reset query cache @@ -65,7 +65,7 @@ endi print ======== step2 sql drop database d1 -sleep 1000 +sleep 500 sql insert into d1.t1 values(now, 2) -x step2 return -1 step2: @@ -73,7 +73,7 @@ step2: print ========= step3 sql create database db1 replica 1 sql reset query cache -sleep 1000 +sleep 500 sql create table db1.tb1 (ts timestamp, i int) sql insert into db1.tb1 values(now, 2) sql select * from db1.tb1 @@ -90,7 +90,7 @@ while $x < 20 sql use $db sql drop database $db - sleep 1000 + sleep 500 sql insert into $tb values(now, -1) -x step4 return -1 step4: @@ -100,7 +100,7 @@ while $x < 20 $tb = tb . $x sql reset query cache - sleep 1000 + sleep 500 sql create database $db replica 1 sql use $db diff --git a/tests/script/general/db/delete_reusevnode.sim b/tests/script/general/db/delete_reusevnode.sim index 5781c384fe9bfd9ac804a3cd85a3e88a1e4bc278..4364a760a6b61be89f00f5a22d19c9ecf9015cb2 100644 --- a/tests/script/general/db/delete_reusevnode.sim +++ b/tests/script/general/db/delete_reusevnode.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======== step1 @@ -38,7 +37,6 @@ if $rows != 0 then endi system sh/stop_dnodes.sh -sleep 3000 system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes diff --git a/tests/script/general/db/delete_reusevnode2.sim b/tests/script/general/db/delete_reusevnode2.sim index 0db2440b3bcd92a9fa252f387be22767ae1f7d7c..c05db6b65a43df2cec9cb410aef69ac23985307e 100644 --- a/tests/script/general/db/delete_reusevnode2.sim +++ b/tests/script/general/db/delete_reusevnode2.sim @@ -4,7 +4,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/db/delete_writing1.sim b/tests/script/general/db/delete_writing1.sim index 93d41b2fa37190e86133c47b087bc0a6a2cb71f1..98e9a3d66f54bc40ac87feb3271403bace0319e8 100644 --- a/tests/script/general/db/delete_writing1.sim +++ b/tests/script/general/db/delete_writing1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database db @@ -44,7 +43,7 @@ while $x < 20 sql create database db sql create table db.tb (ts timestamp, i int) - sleep 3000 + sleep 2000 $x = $x + 1 endw diff --git a/tests/script/general/db/delete_writing2.sim b/tests/script/general/db/delete_writing2.sim index 3ea220cb8ce9ab2e30ab4c57652b3e35add56452..2d1b0c8d9cb4b4bfc8774436accf6b8ff90843da 100644 --- a/tests/script/general/db/delete_writing2.sim +++ b/tests/script/general/db/delete_writing2.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c wallevel -v 0 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db @@ -39,7 +39,7 @@ while $x < 10 sql create database db sql create table db.tb (ts timestamp, i int) - sleep 3000 + sleep 2000 $x = $x + 1 endw diff --git a/tests/script/general/db/dropdnodes.sim b/tests/script/general/db/dropdnodes.sim index c7bbdf73a4b6b6e1f236c7cc920a43def08bd013..8a46d5f9ce405cb84957900c37986cbede044464 100644 --- a/tests/script/general/db/dropdnodes.sim +++ b/tests/script/general/db/dropdnodes.sim @@ -11,7 +11,7 @@ print ========== prepare data system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 @@ -72,7 +72,7 @@ endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql drop dnode $hostname2 sleep 2000 diff --git a/tests/script/general/db/len.sim b/tests/script/general/db/len.sim index 30abcbe10010cd4a6bca8285c9d526133e9ea1f3..561245e666920bbe8b32d3fa3c7999f02f2553f7 100644 --- a/tests/script/general/db/len.sim +++ b/tests/script/general/db/len.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/general/db/nosuchfile.sim b/tests/script/general/db/nosuchfile.sim index 98ac4ec012dc694357878a61ca0dbc11259f0a9e..69db8c0dc53ff39fff9840b3c86deef85e6d6dfa 100644 --- a/tests/script/general/db/nosuchfile.sim +++ b/tests/script/general/db/nosuchfile.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c wallevel -v 2 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ========== step3 sql create database d1 @@ -20,7 +20,7 @@ sql insert into d1.t1 values(now+5s, 31) print ========== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 print ========== step5 sql select * from d1.t1 order by t desc diff --git a/tests/script/general/db/repeat.sim b/tests/script/general/db/repeat.sim index aaa103234d1e3206adce7511e4ac5e105b2dc6c4..b3bbca2d19012e49c3c4210dbbb3b94256cfe278 100644 --- a/tests/script/general/db/repeat.sim +++ b/tests/script/general/db/repeat.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -21,13 +21,13 @@ sql create table d3.t1(ts timestamp, i int) sql create database d4 sql create table d4.t1(ts timestamp, i int) -sleep 3000 +sleep 2000 sql drop database d1 sql drop database d2 sql drop database d3 sql drop database d4 -sleep 3000 +sleep 2000 sql create database d5 sql create table d5.t1(ts timestamp, i int) @@ -41,14 +41,14 @@ sql create table d7.t1(ts timestamp, i int) sql create database d8 sql create table d8.t1(ts timestamp, i int) -sleep 3000 +sleep 2000 sql drop database d5 sql drop database d6 sql drop database d7 sql drop database d8 -sleep 3000 +sleep 2000 sql create database d9; sql create table d9.t1(ts timestamp, i int) diff --git a/tests/script/general/db/show_create_db.sim b/tests/script/general/db/show_create_db.sim index baa7b253e12154c5d320dd6d8411e223ce1cd4e2..edaa971c5cac45d1cb95e48acf84c9c6058901c1 100644 --- a/tests/script/general/db/show_create_db.sim +++ b/tests/script/general/db/show_create_db.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step2 diff --git a/tests/script/general/db/show_create_table.sim b/tests/script/general/db/show_create_table.sim index 83386387093a7c4f778bb5696eea6019fd4691fa..0d7408748a48a41e06dbd1a892aa03595f221383 100644 --- a/tests/script/general/db/show_create_table.sim +++ b/tests/script/general/db/show_create_table.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ===============create three type table diff --git a/tests/script/general/db/tables.sim b/tests/script/general/db/tables.sim index d700bf80689a8d0413b59e13382fdf213ee9c4ec..50b6805ecad44a74a7c421301c7bc6bdc625f21d 100644 --- a/tests/script/general/db/tables.sim +++ b/tests/script/general/db/tables.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step2 diff --git a/tests/script/general/db/vnodes.sim b/tests/script/general/db/vnodes.sim index b01e94206fa33978b359b6c40576284f69b74d87..b123e91ad120e26aad4648794397d2e06b627f8e 100644 --- a/tests/script/general/db/vnodes.sim +++ b/tests/script/general/db/vnodes.sim @@ -15,7 +15,7 @@ system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 100000 print ========== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db blocks 3 cache 1 sql use db diff --git a/tests/script/general/field/2.sim b/tests/script/general/field/2.sim index 28506b7cb82456ce424be5052a1f41269dcd56a3..812301dfbda072e302b8db029c6ee4a514683198 100644 --- a/tests/script/general/field/2.sim +++ b/tests/script/general/field/2.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/3.sim b/tests/script/general/field/3.sim index 453f4968bfe1be218886bf958aaab44a287ba914..a531caaca0471887edf1e573f69cd5fa9839facf 100644 --- a/tests/script/general/field/3.sim +++ b/tests/script/general/field/3.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/4.sim b/tests/script/general/field/4.sim index 243424724fa8a487f8b8ea5394014c5913b097de..c530ff62d1f1a892550a7d755dc35a58c0a9b302 100644 --- a/tests/script/general/field/4.sim +++ b/tests/script/general/field/4.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/5.sim b/tests/script/general/field/5.sim index ca1543b54bf83ee40b077e2542e43e7e5c8e4c03..a676281313abd59e933513b9687d88f91a83ff5d 100644 --- a/tests/script/general/field/5.sim +++ b/tests/script/general/field/5.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/6.sim b/tests/script/general/field/6.sim index 23223e5c1502105b5df13f9d9cad5ad12b08944c..f187d7db108a3b8c787479acf8a1511bbede1125 100644 --- a/tests/script/general/field/6.sim +++ b/tests/script/general/field/6.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/bigint.sim b/tests/script/general/field/bigint.sim index 10060f7422f50ccbb36b118d9dbfbc7729059e3f..c9bda687e19e28adec072bd593b5305eb8e4c2de 100644 --- a/tests/script/general/field/bigint.sim +++ b/tests/script/general/field/bigint.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/binary.sim b/tests/script/general/field/binary.sim index b51f023efe0bb768f90fc2f9e682363d208d0410..36a43272ca094a85c2d95b51495ecaa33fa32c18 100644 --- a/tests/script/general/field/binary.sim +++ b/tests/script/general/field/binary.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/bool.sim b/tests/script/general/field/bool.sim index 4528f79bc79ddb9fb6c2eeeb54c13384eec7df99..d8e613572ace057fd0750e2ea584779a338954c5 100644 --- a/tests/script/general/field/bool.sim +++ b/tests/script/general/field/bool.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/double.sim b/tests/script/general/field/double.sim index 40650cb9bdbc3908892a6e6ee1958fa62c90d6bb..f7dac3192afaf8083df902e6447bcfdbb3e89e5a 100644 --- a/tests/script/general/field/double.sim +++ b/tests/script/general/field/double.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/float.sim b/tests/script/general/field/float.sim index 0ead9fb48a424e5d2c5d2ff3b69a6ecc21fbf540..3ab5602c55f8e5a12c9c978583c7b08891d91757 100644 --- a/tests/script/general/field/float.sim +++ b/tests/script/general/field/float.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/int.sim b/tests/script/general/field/int.sim index a095dc2176ad75c0123c7dcc9a1e0a5a1c75d75f..5c3cec99cfab308559b6450c7c6e7f0487207d10 100644 --- a/tests/script/general/field/int.sim +++ b/tests/script/general/field/int.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/single.sim b/tests/script/general/field/single.sim index 8540608e96aacfe50dc346a0bf4217f8c38db7b7..d572f6df2af770f43f0d72b6a9053bfcc5898561 100644 --- a/tests/script/general/field/single.sim +++ b/tests/script/general/field/single.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/smallint.sim b/tests/script/general/field/smallint.sim index 578de4ea442b87bea61f24db8c24b71efe0e2621..8bd99723217db4800645ef0ca1dffa72b2d8ec95 100644 --- a/tests/script/general/field/smallint.sim +++ b/tests/script/general/field/smallint.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/tinyint.sim b/tests/script/general/field/tinyint.sim index f132b427025d4c706cdcf09b4ede678e36fbaaf8..3a2d7bc44e55432f48a318f1a59bf31bea556986 100644 --- a/tests/script/general/field/tinyint.sim +++ b/tests/script/general/field/tinyint.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim index 98d64ab839df11d17bd6d9036a53fcc087b69ea8..39af990b50d18fbcb6e7972d2adb67a1eb9900e3 100644 --- a/tests/script/general/http/autocreate.sim +++ b/tests/script/general/http/autocreate.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim index 6592c761c6ccaea94f385ac72982115c775f01d7..c5855e5d2953ca36d95cd84a5590c47ef693d406 100644 --- a/tests/script/general/http/chunked.sim +++ b/tests/script/general/http/chunked.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index c7866e5f4ce950292f17a1b4d0f8a49838fc18a1..ca2f62a368fdcbb7d1370d3b6b04ccb5bcca6fa0 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -1,5 +1,5 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -84,7 +84,7 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login print 6-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi @@ -94,7 +94,7 @@ if $system_content != @{"status":"error","code":4387,"desc":"invalid format of A return -1 endi -sleep 3000 +sleep 2000 system_content curl 127.0.0.1:7111/grafana/login/root/taosdata print 8-> $system_content if $system_content != @{"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}@ then diff --git a/tests/script/general/http/grafana_bug.sim b/tests/script/general/http/grafana_bug.sim index 43c52ba75ff72f8567eebaec741f3edee3206d85..ce039af44c22975b164db15852e836175252ed8f 100644 --- a/tests/script/general/http/grafana_bug.sim +++ b/tests/script/general/http/grafana_bug.sim @@ -1,5 +1,5 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 @@ -8,7 +8,7 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ============================ dnode1 start diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim index 9c77567abb9874459a2363a93cb62cb57cf02b1a..ce358d84a177b79f8333c2bb4ce370b79b4e0eca 100644 --- a/tests/script/general/http/gzip.sim +++ b/tests/script/general/http/gzip.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/prepare.sim b/tests/script/general/http/prepare.sim index 0bcb42ad41baa48544e0febba6ccf693f3efb85b..6803643caf7d0b97822c18582597db90f74c5001 100644 --- a/tests/script/general/http/prepare.sim +++ b/tests/script/general/http/prepare.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index 7d1169ca27eac99e740aba3fb8c1f6483791a85b..6ebf5644e7a99593ebac0302c3dcdbc0a27fc8c1 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -1,12 +1,10 @@ system sh/stop_dnodes.sh -sleep 3000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ============================ dnode1 start @@ -41,14 +39,14 @@ print =============== step3 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then return -1 endi print =============== step4 - insert data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.table_rest values('2017-12-25 21:28:51.022', 11)" 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi @@ -56,7 +54,7 @@ print =============== step5 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi @@ -81,4 +79,4 @@ if $system_content != @{"status":"error","code":3,"desc":"Authentication failure return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 94ecb59f75304d99f48ebdb644be432370f86f2a..8094a943f507a702bb56a7d026da672134188b83 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/exec.sh -n dnode1 -s start -#sleep 3000 +#sleep 2000 sql connect print ============================ dnode1 start @@ -15,6 +15,7 @@ print =============== step1 - login system_content curl 127.0.0.1:7111/rest/ print 1-> $system_content if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then + print $system_content return -1 endi @@ -56,19 +57,25 @@ if $system_content != @{"status":"error","code":3,"desc":"Authentication failure endi #8 -system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +system_content curl -H 'Authorization: Beare eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 8-> $system_content if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then return -1 endi +system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +print 8-> $system_content +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then + return -1 +endi + system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 9-> $system_content if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi -sleep 3000 +sleep 2000 system_content curl 127.0.0.1:7111/rest/login/root/taosdata/ print 10-> $system_content @@ -81,13 +88,13 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep0,keep1,keep(D)",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:7111/rest/sql print 12-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi @@ -153,26 +160,26 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d1.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 22-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 23-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[],"rows":0}@ then return -1 endi #24 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 24-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 25-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi @@ -201,32 +208,32 @@ system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl #27 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 27-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d2' 127.0.0.1:7111/rest/sql print 28-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d2.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 29-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi #30 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d2.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 30-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d2.t1 ' 127.0.0.1:7111/rest/sql print 31-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi diff --git a/tests/script/general/http/restful_insert.sim b/tests/script/general/http/restful_insert.sim index f230f98723e041487232c5725b91c4755fe56fcf..90bf7e1b1568f02f3f897bedbbba889be649a169 100644 --- a/tests/script/general/http/restful_insert.sim +++ b/tests/script/general/http/restful_insert.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/restful_limit.sim b/tests/script/general/http/restful_limit.sim index 7d2b6e9a02fce66a3961177c635929302a92387d..c925656b3612e9a52cffc5f53a5a3fb7290ab2e5 100644 --- a/tests/script/general/http/restful_limit.sim +++ b/tests/script/general/http/restful_limit.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index 4018d9661a0a1d7b445fc9eff416bd40f84d2116..f3426971864e6cf510ff6600b7731cf81f64f952 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -1,5 +1,5 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/cfg.sh -n dnode1 -c telegrafUseFieldNum -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -245,7 +245,7 @@ if $system_content != @{"metrics":[{"metric":"win_cpu","stable":"win_cpu","table return -1 endi -sleep 3000 +sleep 2000 print =============== step2 - insert single data system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1564641722000}' 127.0.0.1:7111/telegraf/db/ @@ -285,8 +285,8 @@ system_content curl -u root:taosdata -d 'select count(*) from db.win_cpu' 127.0 print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[3]],"rows":1}@ then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/import/basic.sim b/tests/script/general/import/basic.sim index 07febb2bd52155af1cec70f84a1fe60460815750..50c4059c52c65cf353f5987d44be78cc7649733d 100644 --- a/tests/script/general/import/basic.sim +++ b/tests/script/general/import/basic.sim @@ -26,7 +26,7 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 0 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database ibadb diff --git a/tests/script/general/import/commit.sim b/tests/script/general/import/commit.sim index 36d201e9effaa8398f5087c95b8e03f5893a7846..0aa63b14ff4af7c4c410dd2363b9ad8c98da39a9 100644 --- a/tests/script/general/import/commit.sim +++ b/tests/script/general/import/commit.sim @@ -26,7 +26,7 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 0 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ========= step1 @@ -72,9 +72,9 @@ endi print ========= step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print ========= step4 sql select * from ic2db.tb; diff --git a/tests/script/general/import/large.sim b/tests/script/general/import/large.sim index 5bf05a57fbdb3bf3c8bb123215686ccd5bdd4a8e..3b82c0355aa17266bbd9addbfe13ca0d7235cb8d 100644 --- a/tests/script/general/import/large.sim +++ b/tests/script/general/import/large.sim @@ -26,7 +26,7 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 0 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db diff --git a/tests/script/general/import/replica1.sim b/tests/script/general/import/replica1.sim index d450b3fb49883df2d83b2969d951b981c30b65af..48d5455b7931bfc167112574c8193097aad74334 100644 --- a/tests/script/general/import/replica1.sim +++ b/tests/script/general/import/replica1.sim @@ -27,7 +27,7 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database ir1db days 7 @@ -93,9 +93,9 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use ir1db sql select * from tb; @@ -162,9 +162,9 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use ir1db sql select * from tb; diff --git a/tests/script/general/insert/basic.sim b/tests/script/general/insert/basic.sim index 3f0f25a95ba68c8099c02e6daab933411089a632..c688342fc5cf3c3b77cdfcd45270bc1ccb685f0f 100644 --- a/tests/script/general/insert/basic.sim +++ b/tests/script/general/insert/basic.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/insert_drop.sim b/tests/script/general/insert/insert_drop.sim index 9b68e5a6a6c7e77fc0e591e4950500369a88a650..859263762620b193873e4a355b061f12359c4f2a 100644 --- a/tests/script/general/insert/insert_drop.sim +++ b/tests/script/general/insert/insert_drop.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $tbNum = 10 @@ -43,7 +43,7 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed @@ -69,7 +69,7 @@ endw print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/insert/query_block1_file.sim b/tests/script/general/insert/query_block1_file.sim index 6d6daca7b59dbd1a5fa5f1981aab6ca78691474a..63f46d84f15d813acb1c1166331ab61ba15734a2 100644 --- a/tests/script/general/insert/query_block1_file.sim +++ b/tests/script/general/insert/query_block1_file.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_block1_memory.sim b/tests/script/general/insert/query_block1_memory.sim index bec9190f9b93044d4321b96d6604209cbc78daba..516085f93f8c297b305ef5908010e898673f3a1d 100644 --- a/tests/script/general/insert/query_block1_memory.sim +++ b/tests/script/general/insert/query_block1_memory.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_block2_file.sim b/tests/script/general/insert/query_block2_file.sim index 34da170a9ee6cf3af09de374d8c1da54966a6382..a1fa920c0f389300b621cdaf953a330830282128 100644 --- a/tests/script/general/insert/query_block2_file.sim +++ b/tests/script/general/insert/query_block2_file.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_block2_memory.sim b/tests/script/general/insert/query_block2_memory.sim index 3f2c97a098c35847ae566f6d9fdba0fa85a674d4..9ce0b942d43cb34ad0a8f2e537144283438f3a12 100644 --- a/tests/script/general/insert/query_block2_memory.sim +++ b/tests/script/general/insert/query_block2_memory.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_file_memory.sim b/tests/script/general/insert/query_file_memory.sim index f923ebed13c2f9c592b38dc98ace644fcd8fb671..d43328a65a521654c9d2ae871c76436d8a1251c2 100644 --- a/tests/script/general/insert/query_file_memory.sim +++ b/tests/script/general/insert/query_file_memory.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_multi_file.sim b/tests/script/general/insert/query_multi_file.sim index 8622fa6f9b47ff2b82305fdfff8d187d7b95c275..3b70dd621484f68bf34ed520e366930e67bd8f26 100644 --- a/tests/script/general/insert/query_multi_file.sim +++ b/tests/script/general/insert/query_multi_file.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/tcp.sim b/tests/script/general/insert/tcp.sim index 6f9752087d9d5d099703dd851b55305d65726784..50383efb498a40df5f37542415971eebce615c5c 100644 --- a/tests/script/general/insert/tcp.sim +++ b/tests/script/general/insert/tcp.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database d1; diff --git a/tests/script/general/parser/alter.sim b/tests/script/general/parser/alter.sim index 5c15656f246ffa09217a991ca60e127c74353700..e604d2122e56b4b2d87ca6cd87597abcac02672c 100644 --- a/tests/script/general/parser/alter.sim +++ b/tests/script/general/parser/alter.sim @@ -2,9 +2,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = m_alt_db @@ -114,7 +113,7 @@ endi sql drop table tb sql drop table mt -sleep 500 +sleep 100 ### ALTER TABLE WHILE STREAMING [TBASE271] #sql create table tb1 (ts timestamp, c1 int, c2 nchar(5), c3 int) #sql create table strm as select count(*), avg(c1), first(c2), sum(c3) from tb1 interval(2s) @@ -134,7 +133,7 @@ sleep 500 # return -1 #endi #sql alter table tb1 drop column c3 -#sleep 3000 +#sleep 500 #sql insert into tb1 values (now, 2, 'taos') #sleep 30000 #sql select * from strm @@ -145,9 +144,9 @@ sleep 500 # return -1 #endi #sql alter table tb1 add column c3 int -#sleep 3000 -#sql insert into tb1 values (now, 3, 'taos', 3); #sleep 500 +#sql insert into tb1 values (now, 3, 'taos', 3); +#sleep 100 #sql select * from strm #if $rows != 3 then # return -1 @@ -186,7 +185,7 @@ sql create database $db sql use $db sql create table mt (ts timestamp, c1 int, c2 nchar(7), c3 int) tags (t1 int) sql create table tb using mt tags(1) -sleep 500 +sleep 100 sql insert into tb values ('2018-11-01 16:30:00.000', 1, 'insert', 1) sql alter table mt drop column c3 diff --git a/tests/script/general/parser/alter1.sim b/tests/script/general/parser/alter1.sim index e013242b824a42f9100557f7a9cc267833138e16..26ed029050641d156705cb1a22fe110ce5536ee6 100644 --- a/tests/script/general/parser/alter1.sim +++ b/tests/script/general/parser/alter1.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect sql reset query cache @@ -87,7 +87,7 @@ if $data13 != NULL then return -1 endi -sleep 500 +sleep 100 print ================== insert values into table sql insert into car1 values (now, 1, 1,1 ) (now +1s, 2,2,2,) car2 values (now, 1,3,3) diff --git a/tests/script/general/parser/alter_stable.sim b/tests/script/general/parser/alter_stable.sim index 6b3f3a8f53548979f12707cf381d583674114616..17fc8d984a8899c136cb8451f0fa1e9038bf2cbd 100644 --- a/tests/script/general/parser/alter_stable.sim +++ b/tests/script/general/parser/alter_stable.sim @@ -2,9 +2,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ========== alter_stable.sim diff --git a/tests/script/general/parser/auto_create_tb.sim b/tests/script/general/parser/auto_create_tb.sim index 6deaf92a6c8e713a8fb5281ec2f8e4ca4f2d425f..926eb7547694860810e4018d814637c788787e54 100644 --- a/tests/script/general/parser/auto_create_tb.sim +++ b/tests/script/general/parser/auto_create_tb.sim @@ -4,7 +4,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ======================== dnode1 start @@ -208,11 +208,11 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 sql use $db #### auto create multiple tables diff --git a/tests/script/general/parser/auto_create_tb_drop_tb.sim b/tests/script/general/parser/auto_create_tb_drop_tb.sim index 8a429cf91abff583abd104344ec1ba300ca684bb..b04d02464347ac70b2a0fa74005a72e159b9c78e 100644 --- a/tests/script/general/parser/auto_create_tb_drop_tb.sim +++ b/tests/script/general/parser/auto_create_tb_drop_tb.sim @@ -4,7 +4,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = db @@ -49,7 +49,7 @@ while $t < $tbNum endw print ====== tables created -sleep 500 +sleep 100 sql drop table tb2 $x = 0 diff --git a/tests/script/general/parser/between_and.sim b/tests/script/general/parser/between_and.sim new file mode 100644 index 0000000000000000000000000000000000000000..2e031c4917cf396044a399493ef0be2849baa830 --- /dev/null +++ b/tests/script/general/parser/between_and.sim @@ -0,0 +1,165 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 float, f3 double, f4 bigint, f5 smallint, f6 tinyint, f7 bool, f8 binary(10), f9 nchar(10)) tags (id1 int, id2 float, id3 nchar(10), id4 double, id5 smallint, id6 bigint, id7 binary(10)) + +sql create table tb1 using st2 tags (1,1.0,"1",1.0,1,1,"1"); +sql create table tb2 using st2 tags (2,2.0,"2",2.0,2,2,"2"); +sql create table tb3 using st2 tags (3,3.0,"3",3.0,3,3,"3"); +sql create table tb4 using st2 tags (4,4.0,"4",4.0,4,4,"4"); + +sql insert into tb1 values (now-200s,1,1.0,1.0,1,1,1,true,"1","1") +sql insert into tb1 values (now-100s,2,2.0,2.0,2,2,2,true,"2","2") +sql insert into tb1 values (now,3,3.0,3.0,3,3,3,true,"3","3") +sql insert into tb1 values (now+100s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+200s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") + +sql select tbname,id1 from st2; + +if $rows != 4 then + return -1 +endi + + +sql select * from st2; + +if $rows != 8 then + return -1 +endi + +sql select * from st2 where ts between now-50s and now+450s + +if $rows != 5 then + return -1 +endi + +sql select tbname,id1 from st2 where id1 between 2 and 3; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql select tbname,id2 from st2 where id2 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi + + +sql select tbname,id4 from st2 where id4 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2.000000000 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3.000000000 then + return -1 +endi + + +sql select tbname,id5 from st2 where id5 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql select tbname,id6 from st2 where id6 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql select * from st2 where f1 between 2 and 3 and f2 between 2.0 and 3.0 and f3 between 2.0 and 3.0 and f4 between 2.0 and 3.0 and f5 between 2.0 and 3.0 and f6 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql_error select * from st2 where f7 between 2.0 and 3.0; +sql_error select * from st2 where f8 between 2.0 and 3.0; +sql_error select * from st2 where f9 between 2.0 and 3.0; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index e9e61f35bfda688fcd735085672be95a0fcf626f..dc54add763e5cf34595c479373020c7c4baae63c 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -2,9 +2,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect sql drop database if exists ecdb diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 0cc02d088b38ddb915014f306d89c65e7ff41c6c..3911c2dca6da3f13dd908b4a0e1cc9f6f7279cfa 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect #========================================= setup environment ================================ @@ -105,12 +105,12 @@ run general/parser/col_arithmetic_query.sim #======================================= all in files query ======================================= print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500c +sleep 5000 run general/parser/col_arithmetic_query.sim diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 53e2c98b56e3c395e810e9edbb1d094baaa20f5c..3f1a430e2bb5bab1112708a9925d74d27f030cc5 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -41,10 +41,10 @@ if $data00 != 0.000000000 then return -1 endi -if $data01 != -nan then - print expect -nan, actual: $data01 - return -1 -endi +#if $data01 != -nan then +# print expect -nan, actual: $data01 +# return -1 +#endi if $data10 != 0.666666667 then return -1 @@ -185,9 +185,9 @@ if $data00 != 0.000000000 then return -1 endi -if $data01 != -nan then - return -1 -endi +#if $data01 != -nan then +# return -1 +#endi if $data02 != 0.000000000 then return -1 diff --git a/tests/script/general/parser/columnValue.sim b/tests/script/general/parser/columnValue.sim index 2c03a3552a26ff8c64640f908b15fcc5d194f61e..4e6c6640042a80ffafd008538f2c46e5f43c0318 100644 --- a/tests/script/general/parser/columnValue.sim +++ b/tests/script/general/parser/columnValue.sim @@ -3,9 +3,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ========== columnValues.sim @@ -21,6 +20,7 @@ run general/parser/columnValue_int.sim run general/parser/columnValue_bigint.sim run general/parser/columnValue_float.sim run general/parser/columnValue_double.sim +run general/parser/columnValue_unsign.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/columnValue_bigint.sim b/tests/script/general/parser/columnValue_bigint.sim index 3546ca15ee15eba485f1868d4bae5f4c545218f9..0d89d9e61c20175964304a6b5aa282c074e8b954 100644 --- a/tests/script/general/parser/columnValue_bigint.sim +++ b/tests/script/general/parser/columnValue_bigint.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect sql create database if not exists db sql use db @@ -374,9 +374,9 @@ endi ## case 04: illegal input ################## when overflow, auto set max -sql create table st_bigint_e0 using mt_bigint tags (9223372036854775808) +sql_error create table st_bigint_e0 using mt_bigint tags (9223372036854775808) sql_error create table st_bigint_e0_1 using mt_bigint tags (-9223372036854775808) -sql create table st_bigint_e0_2 using mt_bigint tags (92233720368547758080) +sql_error create table st_bigint_e0_2 using mt_bigint tags (92233720368547758080) sql_error create table st_bigint_e0_3 using mt_bigint tags (-9223372036854775809) #sql_error create table st_bigint_e0 using mt_bigint tags (12.80) truncate integer part #sql_error create table st_bigint_e0 using mt_bigint tags (-11.80) diff --git a/tests/script/general/parser/columnValue_bool.sim b/tests/script/general/parser/columnValue_bool.sim index d68f37590091ab892599d4017a73b6cbb2f28450..3a7dcab2654b372ef59065a59407144ab4fb39c1 100644 --- a/tests/script/general/parser/columnValue_bool.sim +++ b/tests/script/general/parser/columnValue_bool.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/general/parser/columnValue_double.sim b/tests/script/general/parser/columnValue_double.sim index fd2da37838cbabea9c6cee2e992a6904df132c3e..e5ba89158bb98e56428ef7808af6fc4bcd751e48 100644 --- a/tests/script/general/parser/columnValue_double.sim +++ b/tests/script/general/parser/columnValue_double.sim @@ -1,5 +1,5 @@ #### -sleep 500 +sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/general/parser/columnValue_float.sim b/tests/script/general/parser/columnValue_float.sim index 019cf176d0e0f455b5f6b61fcabb501806378c96..c7008d0b132e1d82b32a773555c638e71340b731 100644 --- a/tests/script/general/parser/columnValue_float.sim +++ b/tests/script/general/parser/columnValue_float.sim @@ -1,5 +1,5 @@ #### -sleep 500 +sleep 100 sql connect sql create database if not exists db sql use db @@ -211,23 +211,28 @@ if $data01 != NULL then return -1 endi -sql insert into st_float_6 values (now, 3.40282347e+38) +sql_error insert into st_float_6 values (now, 3.40282347e+38) +sql_error insert into st_float_6 values (now, -3.40282347e+38) + +sql insert into st_float_6 values(now, 340282346638528859811704183484516925440.00000) sql select * from st_float_6 if $rows != 1 then return -1 endi -#if $data01 != 340282346638528859811704183484516925440.00000 then -# print ==== data01:$data01, expect:340282346638528859811704183484516925440.00000 -# return -1 -#endi -sql insert into st_float_7 values (now, -3.40282347e+38) + +if $data01 != 340282346638528859811704183484516925440.00000 then + print ==== data01:$data01, expect:340282346638528859811704183484516925440.00000 + return -1 +endi +sql insert into st_float_7 values (now, -340282346638528859811704183484516925440.00000) sql select * from st_float_7 if $rows != 1 then return -1 endi -#if $data01 != -340282346638528859811704183484516925440.00000 then -# return -1 -#endi +if $data01 != -340282346638528859811704183484516925440.00000 then + return -1 +endi + sql insert into st_float_8 values (now, +100.89) sql select * from st_float_8 if $rows != 1 then @@ -343,111 +348,119 @@ sql select * from st_float_21 if $data01 != NULL then return -1 endi -sql insert into st_float_22 using mt_float tags (127) values (now, 3.40282347e+38) + +sql_error insert into st_float_22 using mt_float tags (127) values (now, 3.40282347e+38) + +sql insert into st_float_22 using mt_float tags (127) values (now, 340282346638528859811704183484516925440.00000) sql select tagname from st_float_22 -#if $data00 != 127 then -# return -1 -#endi -sql select * from st_float_22 -#if $data01 != 127 then -# return -1 -#endi -sql insert into st_float_23 using mt_float tags (-127) values (now, -3.40282347e+38) +if $data00 != 127.00000 then + print expect 127.00000, actual: $data00 + return -1 +endi + +sql select tbname, tagname from st_float_22 +if $data01 != 127.00000 then + return -1 +endi + +sql insert into st_float_23 using mt_float tags (-127) values (now, -340282346638528859811704183484516925440.00000) sql select tagname from st_float_23 -#if $data00 != -127 then -# return -1 -#endi -sql select * from st_float_23 -#if $data01 != -127 then -# return -1 -#endi +if $data00 != -127.00000 then + return -1 +endi + sql insert into st_float_24 using mt_float tags (10) values (now, 10) sql select tagname from st_float_24 -#if $data00 != 10 then -# return -1 -#endi +if $data00 != 10.00000 then + return -1 +endi sql select * from st_float_24 -#if $data01 != 10 then -# return -1 -#endi +if $data01 != 10.00000 then + return -1 +endi + sql insert into st_float_25 using mt_float tags ("-0") values (now, "-0") sql select tagname from st_float_25 -#if $data00 != 0 then -# return -1 -#endi +if $data00 != -0.00000 then + print expect -0.00000, actual: $data00 + return -1 +endi sql select * from st_float_25 -#if $data01 != 0 then -# return -1 -#endi +if $data01 != -0.00000 then + return -1 +endi sql insert into st_float_26 using mt_float tags ('123') values (now, '12.3') sql select tagname from st_float_26 -#if $data00 != 123 then -# return -1 -#endi +if $data00 != 123.00000 then + print expect 123.00000, actual: $data00 + return -1 +endi sql select * from st_float_26 -#if $data01 != 123 then -# return -1 -#endi +if $data01 != 12.30000 then + return -1 +endi sql insert into st_float_27 using mt_float tags (+056) values (now, +0005.6) sql select tagname from st_float_27 -#if $data00 != 56 then -# return -1 -#endi +if $data00 != 56.00000 then + print expect 56.00000, actual:$data00 + return -1 +endi sql select * from st_float_27 -#if $data01 != 56 then -# return -1 -#endi +if $data01 != 5.60000 then + return -1 +endi sql insert into st_float_28 using mt_float tags (-056) values (now, -005.6) sql select tagname from st_float_28 -#if $data00 != -56 then -# return -1 -#endi +if $data00 != -56.00000 then + return -1 +endi sql select * from st_float_28 -#if $data01 != -56 then -# return -1 -#endi +if $data01 != -5.60000 then + return -1 +endi ### case 03: alter tag values -#sql alter table st_float_0 set tag tagname=3.40282347e+38 -#sql select tagname from st_float_0 -##if $data00 != 127 then -## return -1 -##endi -#sql alter table st_float_0 set tag tagname=-3.40282347e+38 -#sql select tagname from st_float_0 -##if $data00 != -127 then -## return -1 -##endi -#sql alter table st_float_0 set tag tagname=+10.340 -#sql select tagname from st_float_0 -##if $data00 != 100 then -## return -1 -##endi -#sql alter table st_float_0 set tag tagname=-33.87 -#sql select tagname from st_float_0 -##if $data00 != -33 then -## return -1 -##endi -#sql alter table st_float_0 set tag tagname='+9.8' -#sql select tagname from st_float_0 -##if $data00 != 98 then -## return -1 -##endi -#sql alter table st_float_0 set tag tagname='-07.6' -#sql select tagname from st_float_0 -##if $data00 != -76 then -## return -1 -##endi -#sql alter table st_float_0 set tag tagname=+0012.871 -#sql select tagname from st_float_0 -##if $data00 != 12 then -## return -1 -##endi -#sql alter table st_float_0 set tag tagname=-00063.582 -#sql select tagname from st_float_0 -##if $data00 != -63 then -## return -1 -##endi +sql alter table st_float_0 set tag tagname=340282346638528859811704183484516925440.00000 +sql select tagname from st_float_0 +if $data00 != 340282346638528859811704183484516925440.00000 then + return -1 +endi + +sql alter table st_float_0 set tag tagname=-340282346638528859811704183484516925440.00000 +sql select tagname from st_float_0 +if $data00 != -340282346638528859811704183484516925440.00000 then + return -1 +endi +sql alter table st_float_0 set tag tagname=+10.340 +sql select tagname from st_float_0 +if $data00 != 10.34000 then + return -1 +endi +sql alter table st_float_0 set tag tagname=-33.87 +sql select tagname from st_float_0 +if $data00 != -33.87000 then + return -1 +endi +sql alter table st_float_0 set tag tagname='+9.8' +sql select tagname from st_float_0 +if $data00 != 9.80000 then + return -1 +endi +sql alter table st_float_0 set tag tagname='-07.6' +sql select tagname from st_float_0 +if $data00 != -7.60000 then + return -1 +endi +sql alter table st_float_0 set tag tagname=+0012.871 +sql select tagname from st_float_0 +if $data00 != 12.87100 then + return -1 +endi +sql alter table st_float_0 set tag tagname=-00063.582 +sql select tagname from st_float_0 +if $data00 != -63.58200 then + return -1 +endi ## case 04: illegal input sql_error create table st_float_e0 using mt_float tags (3.50282347e+38) diff --git a/tests/script/general/parser/columnValue_int.sim b/tests/script/general/parser/columnValue_int.sim index 1f84df5ca05ef70e95c2db03efe0a670abaff8da..5536293ed2d0dcac847829ba03891621cc62eab9 100644 --- a/tests/script/general/parser/columnValue_int.sim +++ b/tests/script/general/parser/columnValue_int.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/general/parser/columnValue_smallint.sim b/tests/script/general/parser/columnValue_smallint.sim index af5c5818717b6c2f01e4726975ceaee4703cdbfb..98b83fd0e19e07d9e95f355cbdebc7b84341891c 100644 --- a/tests/script/general/parser/columnValue_smallint.sim +++ b/tests/script/general/parser/columnValue_smallint.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/general/parser/columnValue_tinyint.sim b/tests/script/general/parser/columnValue_tinyint.sim index 3efe52cc91f2ecb1f22000a3f212ec491b9a16c1..7b0cad23dfdfe6970cf9bd24f2dc8673b266b10c 100644 --- a/tests/script/general/parser/columnValue_tinyint.sim +++ b/tests/script/general/parser/columnValue_tinyint.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect sql create database if not exists db sql use db diff --git a/tests/script/general/parser/columnValue_unsign.sim b/tests/script/general/parser/columnValue_unsign.sim new file mode 100644 index 0000000000000000000000000000000000000000..8e44ccb5facf074691f3aeeb7c60099ab6ef691f --- /dev/null +++ b/tests/script/general/parser/columnValue_unsign.sim @@ -0,0 +1,176 @@ +sleep 100 +sql connect +sql create database if not exists db +sql use db + +sql drop table if exists mt_unsigned; + +sql create table mt_unsigned (ts timestamp, a tinyint unsigned, b smallint unsigned, c int unsigned, d bigint unsigned, e tinyint, f smallint, g int, h bigint, j bool) tags (t1 tinyint unsigned, t2 smallint unsigned, t3 int unsigned, t4 bigint unsigned, t5 tinyint, t6 smallint, t7 int, t8 bigint); +sql create table mt_unsigned_1 using mt_unsigned tags(0, 0, 0, 0, 0, 0, 0, 0); + +sql alter table mt_unsigned_1 set tag t1=138; +sql alter table mt_unsigned_1 set tag t2=32769; +sql alter table mt_unsigned_1 set tag t3=294967295; +sql alter table mt_unsigned_1 set tag t4=446744073709551615; +sql select t1,t2,t3,t4 from mt_unsigned_1 +if $rows != 1 then + return -1 +endi + +print $data00, $data01, $data02, $data03 + +if $data00 != 138 then + print expect 138, actual: $data00 + return -1 +endi + +if $data01 != 32769 then +return -1 +endi + +if $data02 != 294967295 then +return -1 +endi + +if $data03 != 446744073709551615 then +return -1 +endi + +sql_error sql alter table mt_unsigned_1 set tag t1 = 999; +sql_error sql alter table mt_unsigned_1 set tag t2 = 95535; +sql_error sql alter table mt_unsigned_1 set tag t3 = 8294967295l; +sql_error sql alter table mt_unsigned_1 set tag t4 = 19446744073709551615; + +sql_error create table mt_unsigned_2 using mt_unsigned tags(-1, 0, 0, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_3 using mt_unsigned tags(0, -1, 0, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_4 using mt_unsigned tags(0, 0, -1, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_5 using mt_unsigned tags(0, 0, 0, -1, 0, 0, 0, 0); + +sql_error create table mt_unsigned_2 using mt_unsigned tags(255, 0, 0, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_3 using mt_unsigned tags(0, 65535, 0, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_4 using mt_unsigned tags(0, 0, 4294967295, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_5 using mt_unsigned tags(0, 0, 0, 18446744073709551615, 0, 0, 0, 0); + +sql_error create table mt_unsigned_2 using mt_unsigned tags(999, 0, 0, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_3 using mt_unsigned tags(0, 95535, 0, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_4 using mt_unsigned tags(0, 0, 5294967295l, 0, 0, 0, 0, 0); +sql_error create table mt_unsigned_5 using mt_unsigned tags(0, 0, 0, 28446744073709551615u, 0, 0, 0, 0); + +sql alter table mt_unsigned_1 set tag t1=NULL; +sql alter table mt_unsigned_1 set tag t2=NULL; +sql alter table mt_unsigned_1 set tag t3=NULL; +sql alter table mt_unsigned_1 set tag t4=NULL; +sql select t1,t2,t3,t4 from mt_unsigned_1 +if $rows != 1 then + return -1; +endi + +if $data00 != NULL then + print expect NULL, actual: $data00 + return -1 +endi + +if $data01 != NULL then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data03 != NULL then + return -1 +endi + +sql insert into mt_unsigned_1 values(now, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +sql insert into mt_unsigned_1 values(now+1s, 1, 2, 3, 4, 5, 6, 7, 8, 9); + +sql_error insert into mt_unsigned_1 values(now, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +sql_error insert into mt_unsigned_1 values(now, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +sql_error insert into mt_unsigned_1 values(now, NULL, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL); +sql_error insert into mt_unsigned_1 values(now, NULL, NULL, NULL, -1, NULL, NULL, NULL, NULL, NULL); +sql_error insert into mt_unsigned_1 values(now, 255, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +sql_error insert into mt_unsigned_1 values(now, NULL, 65535, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +sql_error insert into mt_unsigned_1 values(now, NULL, NULL, 4294967295, NULL, NULL, NULL, NULL, NULL, NULL); +sql_error insert into mt_unsigned_1 values(now, NULL, NULL, NULL, 18446744073709551615, NULL, NULL, NULL, NULL, NULL); +sql select count(a),count(b),count(c),count(d), count(e) from mt_unsigned_1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select a+b+c from mt_unsigned_1 where a is null; +if $rows != 1 then + return -1 +endi + +if $data00 != NULL then + print expect NULL, actual:$data00 + return -1 +endi + +sql select count(*), a from mt_unsigned_1 group by a; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select count(*), b from mt_unsigned_1 group by b; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +sql select count(*), c from mt_unsigned_1 group by c; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select count(*), d from mt_unsigned_1 group by d; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 4 then + return -1 +endi + +## todo insert more rows and chec it +sql select first(a),count(b),last(c),sum(b),spread(d),avg(c),min(b),max(a),stddev(a) from mt_unsigned_1; +if $rows != 1 then + return -1 +endi + diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index c798bf9d7cc21f2c91c8c14e8debe22ad253b72b..4085ef620d26ac7c869d6f2a298022ac2fe19564 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxTablesperVnode -v 100 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = sc_db @@ -82,12 +82,12 @@ endw print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 -system sh/exec.sh -n dnode1 -s start sleep 500 +system sh/exec.sh -n dnode1 -s start +sleep 100 print ================== server restart completed sql connect -sleep 500 +sleep 100 print ====== select from table and check num of rows returned sql use $db diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim index 3b7f24b6d900dc1f130f91a138da81fd708b16fe..ea1cc17a6a779391c8dfa064bf98bf70715d9e70 100644 --- a/tests/script/general/parser/create_db.sim +++ b/tests/script/general/parser/create_db.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ======================== dnode1 start @@ -169,8 +169,8 @@ sql_error create database $db cache 0 sql_error create database $db ctime 29 sql_error create database $db ctime 40961 -# wal {1, 2} -sql_error create database $db wal 0 +# wal {0, 2} +#sql_error create database $db wal 0 sql_error create database $db wal -1 sql_error create database $db wal 3 diff --git a/tests/script/general/parser/create_mt.sim b/tests/script/general/parser/create_mt.sim index e11f32276187d15334193af688b3c50604df2b32..9278fbfba415a89f79763b5fb8bbda67b0eb89c5 100644 --- a/tests/script/general/parser/create_mt.sim +++ b/tests/script/general/parser/create_mt.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ======================== dnode1 start diff --git a/tests/script/general/parser/create_tb.sim b/tests/script/general/parser/create_tb.sim index 609aad2adbc9611d33c595d33191e5b8a26b3359..48b7a0fb1994f192cf984b37665fdc6281e9937f 100644 --- a/tests/script/general/parser/create_tb.sim +++ b/tests/script/general/parser/create_tb.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ======================== dnode1 start diff --git a/tests/script/general/parser/create_tb_with_tag_name.sim b/tests/script/general/parser/create_tb_with_tag_name.sim new file mode 100644 index 0000000000000000000000000000000000000000..bbd5fc11e1eb662053860bca4f0d4210c1d1fbbc --- /dev/null +++ b/tests/script/general/parser/create_tb_with_tag_name.sim @@ -0,0 +1,162 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st2 (ts timestamp, f1 int) tags (id int, t1 int, t2 nchar(4), t3 double) + + +sql insert into tb1 using st2 (id, t1) tags(1,2) values (now, 1) + +sql select id,t1,t2,t3 from tb1 + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != NULL then + return -1 +endi +if $data03 != NULL then + return -1 +endi + +sql create table tb2 using st2 (t2,t3) tags ("12",22.0) + +sql select id,t1,t2,t3 from tb2; + +if $rows != 1 then + return -1 +endi + +if $data00 != NULL then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 12 then + return -1 +endi +if $data03 != 22.000000000 then + return -1 +endi + + +sql create table tb3 using st2 tags (1,2,"3",33.0); + +sql select id,t1,t2,t3 from tb3; + + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 3 then + return -1 +endi +if $data03 != 33.000000000 then + return -1 +endi + +sql insert into tb4 using st2 tags(1,2,"33",44.0) values (now, 1); + +sql select id,t1,t2,t3 from tb4; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 33 then + return -1 +endi +if $data03 != 44.000000000 then + return -1 +endi + +sql_error create table tb5 using st2() tags (3,3,"3",33.0); + +sql_error create table tb6 using st2 (id,t1) tags (3,3,"3",33.0); + +sql_error create table tb7 using st2 (id,t1) tags (3); + +sql_error create table tb8 using st2 (ide) tags (3); + +sql_error create table tb9 using st2 (id); + +sql_error create table tb10 using st2 (id t1) tags (1,1); + +sql_error create table tb10 using st2 (id,,t1) tags (1,1,1); + +sql_error create table tb11 using st2 (id,t1,) tags (1,1,1); + +sql create table tb12 using st2 (t1,id) tags (2,1); + +sql select id,t1,t2,t3 from tb12; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != NULL then + return -1 +endi +if $data03 != NULL then + return -1 +endi + +sql create table tb13 using st2 ("t1",'id') tags (2,1); + +sql select id,t1,t2,t3 from tb13; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != NULL then + return -1 +endi +if $data03 != NULL then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/dbtbnameValidate.sim b/tests/script/general/parser/dbtbnameValidate.sim index fd40ecc3f758affa3ca401813132b830a241a049..5fc67334c4b03329a7807c448eea38ebb4f2bdce 100644 --- a/tests/script/general/parser/dbtbnameValidate.sim +++ b/tests/script/general/parser/dbtbnameValidate.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ========== db name and table name check in create and drop, describe @@ -12,25 +12,25 @@ sql create database 'abc123' sql create database '_ab1234' sql create database 'ABC123' sql create database '_ABC123' -sql create database 'aABb123 ' -sql create database ' xyz ' -sql create database ' XYZ ' +sql_error create database 'aABb123 ' +sql_error create database ' xyz ' +sql_error create database ' XYZ ' sql use 'abc123' sql use '_ab1234' sql use 'ABC123' sql use '_ABC123' -sql use 'aABb123' -sql use ' xyz ' -sql use ' XYZ ' +sql_error use 'aABb123' +sql_error use ' xyz ' +sql_error use ' XYZ ' sql drop database 'abc123' sql drop database '_ab1234' -sql drop database 'ABC123' +sql_error drop database 'ABC123' sql drop database '_ABC123' -sql drop database 'aABb123' -sql drop database ' xyz ' -sql drop database ' XYZ ' +sql_error drop database 'aABb123' +sql_error drop database ' xyz ' +sql_error drop database ' XYZ ' sql use abc @@ -67,9 +67,9 @@ sql describe mt sql describe sub_001 sql describe sub_dy_tbl -sql_error describe Dd -sql_error describe FF -sql_error describe gG +sql describe Dd +sql describe FF +sql describe gG sql drop table abc.cc sql drop table 'abc.Dd' @@ -119,4 +119,4 @@ if $rows != 4 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index 9851a4e7fcfea026271729cbe7436c47e184df40..aac79e1a3ca5a52a89d4f29ba43efc7be228c0f9 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = m_fl_db @@ -848,10 +848,7 @@ if $rows != 12 then return -1 endi -print =====================>td-1442 -sql_error select count(*) from m_fl_tb0 interval(1s) fill(prev); - -print =====================> aggregation + arithmetic + fill +print =====================> aggregation + arithmetic + fill, need to add cases TODO #sql select avg(cpu_taosd) - first(cpu_taosd) from dn1 where ts<'2020-11-13 11:00:00' and ts>'2020-11-13 10:50:00' interval(10s) fill(value, 99) #sql select count(*), first(k), avg(k), avg(k)-first(k) from tm0 where ts>'2020-1-1 1:1:1' and ts<'2020-1-1 1:02:59' interval(10s) fill(value, 99); #sql select count(*), first(k), avg(k), avg(k)-first(k) from tm0 where ts>'2020-1-1 1:1:1' and ts<'2020-1-1 1:02:59' interval(10s) fill(NULL); @@ -1044,6 +1041,17 @@ if $data12 != 1 then return -1 endi +print =====================>td-1442, td-2190 , no time range for fill option +sql_error select count(*) from m_fl_tb0 interval(1s) fill(prev); + +sql_error select min(c3) from m_fl_mt0 interval(10a) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10s) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10m) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10h) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10d) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10w) fill(value, 20) +sql_error select max(c3) from m_fl_mt0 interval(1n) fill(prev) +sql_error select min(c3) from m_fl_mt0 interval(1y) fill(value, 20) print =============== clear #sql drop database $db diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index 83eb98c465927beb70ce403491dcd100f2556c7d..a9547b8a9408a52aa3cda64301326b81c2e4655f 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = fl1_db diff --git a/tests/script/general/parser/fill_us.sim b/tests/script/general/parser/fill_us.sim index dc8ee8659d6c3b99e8103e3ee0a622a10e7bc7df..a429df059bad3cd36301ecd4685db89d74090ba0 100644 --- a/tests/script/general/parser/fill_us.sim +++ b/tests/script/general/parser/fill_us.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = m_fl_db diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index 773f92afcfd1fb126f137befd9de1107b050356f..aeff740a5f790533e26fda99205d3cffc876deb1 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxTablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = first_db @@ -77,11 +77,11 @@ run general/parser/first_last_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 run general/parser/first_last_query.sim @@ -106,11 +106,11 @@ while $x < 5000 endw system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 1000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 sql use test sql select count(*), last(ts) from tm0 interval(1s) diff --git a/tests/script/general/parser/first_last_query.sim b/tests/script/general/parser/first_last_query.sim index 52d888b04d1b055bbbc3c8509832c96579e4320f..2d08759f3fb0bdd78a3163e335334a49504ec861 100644 --- a/tests/script/general/parser/first_last_query.sim +++ b/tests/script/general/parser/first_last_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = first_db @@ -37,6 +37,7 @@ if $data02 != 0 then endi print data03 = $data03 if $data03 != 0.00000 then + print expect 0.00000, actual: $data03 return -1 endi if $data04 != 0.000000000 then diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 34e9844f711313d9f7c1c2380177bfc2c7aa4740..af16bfd4f18a40d9964b6f4f7d4f6ea0840937a2 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -2,9 +2,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = m_func_db @@ -22,7 +21,7 @@ $db = $dbPrefix . $i $mt = $mtPrefix . $i sql drop database if exists $db -sql create database $db +sql create database $db keep 36500 sql use $db print =====================================> test case for twa in single block @@ -111,7 +110,7 @@ if $rows != 2 then return -1 endi -if $data00 != @15-08-18 00:06:00.00@ then +if $data00 != @15-08-18 00:06:00.000@ then return -1 endi @@ -219,10 +218,565 @@ if $data02 != 6 then return -1 endi +sql select twa(k) from t1 where ts>'2015-8-18 00:00:00' and ts<'2015-8-18 00:00:1' +if $rows != 0 then + return -1 +endi + sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts asc sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts desc -#todo add test case while column filte exists. +#todo add test case while column filter exists for twa query + +#sql select count(*),TWA(k) from tm0 where ts>='1970-1-1 13:43:00' and ts<='1970-1-1 13:44:10' interval(9s) + +sql create table tm0 (ts timestamp, k float); +sql insert into tm0 values(100000000, 5); +sql insert into tm0 values(100003000, -9); +sql select twa(k) from tm0 where ts td-2610 +sql select twa(k)from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-9 18:11:17.098' +if $rows != 0 then + return -1 +endi + +print =====================> td-2609 +sql select apercentile(k, 50) from tm1 where ts>='2020-10-30 18:11:56.680' and ts<='2020-12-09 18:11:17.098' +if $rows != 1 then + return -1 +endi + +if $data00 != -1000.000000000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 1000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect +sleep 100 + +sql use m_func_db0 + +print =====================> td-2583 +sql select min(k) from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-20 18:11:49.412' +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + print expect 1, actual: $data00 + return -1 +endi + +print =====================> td-2601 +sql select count(*) from tm1 where ts<='2020-6-1 00:00:00' and ts>='2020-1-1 00:00:00' interval(1n) fill(NULL) +if $rows != 0 then + return -1 +endi + +print =====================> td-2615 +sql select last(ts) from tm1 interval(17a) limit 776 offset 3 +if $rows != 3 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) limit 1000 offset 4 +if $rows != 2 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) order by ts desc limit 1000 offset 0 +if $rows != 6 then + return -1 +endi + +print ==================> td-2624 +sql create table tm2(ts timestamp, k int, b binary(12)); +sql insert into tm2 values('2011-01-02 18:42:45.326', -1,'abc'); +sql insert into tm2 values('2020-07-30 17:44:06.283', 0, null); +sql insert into tm2 values('2020-07-30 17:44:19.578', 9999999, null); +sql insert into tm2 values('2020-07-30 17:46:06.417', NULL, null); +sql insert into tm2 values('2020-11-09 18:42:25.538', 0, null); +sql insert into tm2 values('2020-12-29 17:43:11.641', 0, null); +sql insert into tm2 values('2020-12-29 18:43:17.129', 0, null); +sql insert into tm2 values('2020-12-29 18:46:19.109', NULL, null); +sql insert into tm2 values('2021-01-03 18:40:40.065', 0, null); + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s); +if $rows != 6 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data02 != @11-01-02 18:42:45.326@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi + +if $data21 != 0.000000000 then + return -1 +endi + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s) order by ts desc; +if $rows != 6 then + return -1 +endi + +sql select twa(k),first(ts),count(k),first(k) from tm2 interval(17s) limit 20 offset 0; +if $rows != 9 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi + +print =================>td-2610 +sql select stddev(k) from tm2 where ts='2020-12-29 18:46:19.109' +if $rows != 0 then + print expect 0, actual:$rows + return -1 +endi + +sql select twa(k) from tm2 where ts='2020-12-29 18:46:19.109' +if $rows != 0 then + return -1 +endi + +print ========================> TD-1787 +sql create table cars(ts timestamp, c int) tags(id int); +sql create table car1 using cars tags(1); +sql create table car2 using cars tags(2); +sql insert into car1 (ts, c) values (now,1) car2(ts, c) values(now, 2); +sql drop table cars; +sql create table cars(ts timestamp, c int) tags(id int); +sql create table car1 using cars tags(1); +sql create table car2 using cars tags(2); +sql insert into car1 (ts, c) values (now,1) car2(ts, c) values(now, 2); + +print ========================> TD-2700 +sql create table tx(ts timestamp, k int); +sql insert into tx values(1500000001000, 0); +sql select sum(k) from tx interval(1d) sliding(1h); +if $rows != 24 then + print expect 24, actual:$rows + return -1 +endi + +print ========================> TD-2740 +sql drop table if exists m1; +sql create table m1(ts timestamp, k int) tags(a int); +sql create table tm10 using m1 tags(0); +sql create table tm11 using m1 tags(1); +sql create table tm12 using m1 tags(2); +sql create table tm13 using m1 tags(3); +sql insert into tm10 values('2020-1-1 1:1:1', 0); +sql insert into tm11 values('2020-1-5 1:1:1', 0); +sql insert into tm12 values('2020-1-7 1:1:1', 0); +sql insert into tm13 values('2020-1-1 1:1:1', 0); +sql select count(*) from m1 where ts='2020-1-1 1:1:1' interval(1h) group by tbname; +if $rows != 2 then + return -1 +endi + +sql drop table m1; +sql drop table if exists tm1; +sql drop table if exists tm2; +sql create table m1(ts timestamp, k double, b double, c int, d smallint, e int unsigned) tags(a int); +sql create table tm1 using m1 tags(1); +sql create table tm2 using m1 tags(2); +sql insert into tm1 values('2021-01-27 22:22:39.294', 1, 10, NULL, 110, 123) ('2021-01-27 22:22:40.294', 2, 20, NULL, 120, 124) ('2021-01-27 22:22:41.294', 3, 30, NULL, 130, 125)('2021-01-27 22:22:43.294', 4, 40, NULL, 140, 126)('2021-01-27 22:22:44.294', 5, 50, NULL, 150, 127); +sql insert into tm2 values('2021-01-27 22:22:40.688', 5, 101, NULL, 210, 321) ('2021-01-27 22:22:41.688', 5, 102, NULL, 220, 322) ('2021-01-27 22:22:42.688', 5, 103, NULL, 230, 323)('2021-01-27 22:22:43.688', 5, 104, NULL, 240, 324)('2021-01-27 22:22:44.688', 5, 105, NULL, 250, 325)('2021-01-27 22:22:45.688', 5, 106, NULL, 260, 326); +sql select stddev(k) from m1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.378704626 then + return -1 +endi + +sql select stddev(c) from m1 +if $rows != 0 then + return -1 +endi + +sql select stddev(k), stddev(c) from m1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.378704626 then + return -1 +endi + +if $data01 != NULL then + return -1; +endi + +sql select stddev(b),stddev(b),stddev(k) from m1; +if $rows != 1 then + return -1 +endi + +if $data00 != 37.840465463 then + return -1 +endi + +if $data01 != 37.840465463 then + return -1 +endi + +if $data02 != 1.378704626 then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 group by a +if $rows != 2 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data10 != 0.000000000 then + return -1 +endi + +if $data11 != 1.707825128 then + return -1 +endi + +if $data12 != 2 then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 where a= 1 group by a +if $rows != 1 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 group by tbname +if $rows != 2 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != @tm1@ then + return -1 +endi + +if $data10 != 0.000000000 then + return -1 +endi + +if $data11 != 1.707825128 then + return -1 +endi + +if $data12 != @tm2@ then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 group by tbname,a +if $rows != 2 then + return -1 +endi + +sql select stddev(k), stddev(b), stddev(c) from m1 group by tbname,a +if $rows != 2 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data03 != @tm1@ then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + +if $data10 != 0.000000000 then + return -1 +endi + +if $data11 != 1.707825128 then + return -1 +endi + +if $data12 != NULL then + return -1 +endi + +if $data13 != @tm2@ then + return -1 +endi + +if $data14 != 2 then + return -1 +endi + +sql select stddev(k), stddev(b), stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data01 != 0.000000000 then + return -1 +endi + +if $data02 != 0.000000000 then + return -1 +endi + +if $data03 != NULL then + return -1 +endi + +if $data04 != @tm1@ then + return -1 +endi + +if $data05 != 1 then + return -1 +endi + +if $data11 != 1.118033989 then + return -1 +endi + +if $data12 != 11.180339887 then + return -1 +endi + +if $data13 != NULL then + return -1 +endi + +if $data14 != @tm1@ then + return -1 +endi + +if $data22 != 1.707825128 then + return -1 +endi + +if $data23 != NULL then + return -1 +endi + +if $data24 != @tm2@ then + return -1 +endi + +if $data25 != 2 then + return -1 +endi + +sql select count(*), first(b), stddev(b), stddev(c) from m1 interval(10s) group by a +if $rows != 3 then + return -1 +endi + +if $data00 != @21-01-27 22:22:30.000@ then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 10.000000000 then + return -1 +endi + +if $data03 != 0.000000000 then + return -1 +endi + +if $data04 != NULL then + return -1 +endi + +if $data05 != 1 then + return -1 +endi + +if $data12 != 20.000000000 then + return -1 +endi + +if $data13 != 11.180339887 then + return -1 +endi + +if $data14 != NULL then + return -1 +endi + +if $data23 != 1.707825128 then + return -1 +endi + +sql select count(*), first(b), stddev(b), stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data23 != 1.707825128 then + return -1 +endi + +if $data25 != @tm2@ then + return -1 +endi + +sql select count(*), stddev(b), stddev(b)+20, stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data02 != 0.000000000 then + return -1 +endi + +if $data03 != 20.000000000 then + return -1 +endi + +if $data13 != 31.180339887 then + return -1 +endi + +if $data14 != NULL then + return -1 +endi + +sql select count(*), first(b), stddev(b)+first(b), stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data02 != 10.000000000 then + return -1 +endi + +if $data03 != 10.000000000 then + return -1 +endi + +if $data12 != 20.000000000 then + return -1 +endi + +if $data13 != 31.180339887 then + return -1 +endi + +if $data22 != 101.000000000 then + return -1 +endi + +if $data23 != 102.707825128 then + return -1 +endi + +sql select stddev(e),stddev(k) from m1 where a=1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 1.414213562 then + return -1 +endi + +sql create stable st1 (ts timestamp, f1 int, f2 int) tags (id int); +sql create table tb1 using st1 tags(1); + +sql insert into tb1 values (now, 1, 1); + +sql select stddev(f1) from st1 group by f1; + +if $rows != 1 then + return -1 +endi + + +if $data00 != 0.000000000 then + return -1 +endi -select count(*),TWA(k) from tm0 where ts>='1970-1-1 13:43:00' and ts<='1970-1-1 13:44:10' interval(9s) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 44bae6f242f12e5a6430df64520631d0ac3af073..19b14e327c0cd180957748b656b8ad15c0ad35cd 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = group_db @@ -606,6 +606,10 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; +print =================>TD-2665 +sql_error create table txx as select avg(c) as t from st; +sql_error create table txx1 as select avg(c) as t from t1; + print =================>TD-2236 sql select first(ts),last(ts) from t1 group by c; if $rows != 4 then diff --git a/tests/script/general/parser/import.sim b/tests/script/general/parser/import.sim index 6da2483738a6d5c805e3a29129183845e64c4f22..d626f4fa74eb3f53a9f9118800494a05320678a7 100644 --- a/tests/script/general/parser/import.sim +++ b/tests/script/general/parser/import.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = impt_db @@ -25,15 +25,15 @@ sql use $db sql create table tb (ts timestamp, c1 int, c2 timestamp) sql insert into tb values ('2019-05-05 11:30:00.000', 1, now) sql insert into tb values ('2019-05-05 12:00:00.000', 1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-05 11:00:00.000', -1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-05 11:59:00.000', -1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-04 08:00:00.000', -1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-04 07:59:00.000', -1, now) -sleep 2000 +sleep 500 sql select * from tb if $rows != 6 then @@ -60,11 +60,11 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 2000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 sql use $db sql select * from tb diff --git a/tests/script/general/parser/import_commit1.sim b/tests/script/general/parser/import_commit1.sim index 9c5144a630cbd0df20e8842211c1d3d3e5d5cbb0..f330fe4cd94b1cba82ea985b9b4841245753e099 100644 --- a/tests/script/general/parser/import_commit1.sim +++ b/tests/script/general/parser/import_commit1.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = ic_db @@ -40,7 +40,7 @@ while $x < $rowNum endw print ====== tables created -sleep 3000 +sleep 500 $ts = $ts0 + $delta $ts = $ts + 1 diff --git a/tests/script/general/parser/import_commit2.sim b/tests/script/general/parser/import_commit2.sim index 000394386ec169bba4b907969d73878760584eb6..47b30acb49111da69c1c0b8b1927fd7f13f0c31c 100644 --- a/tests/script/general/parser/import_commit2.sim +++ b/tests/script/general/parser/import_commit2.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = ic_db @@ -39,7 +39,7 @@ while $x < $rowNum endw print ====== tables created -sleep 3000 +sleep 500 $ts = $ts0 + $delta $ts = $ts + 1 diff --git a/tests/script/general/parser/import_commit3.sim b/tests/script/general/parser/import_commit3.sim index 997a4a22aada1d3f02becd8e02f82c35a41a6175..1e041375de3da7fc2a5c17ae40caa1251b5d9d1b 100644 --- a/tests/script/general/parser/import_commit3.sim +++ b/tests/script/general/parser/import_commit3.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = ic_db @@ -39,7 +39,7 @@ while $x < $rowNum endw print ====== tables created -sleep 3000 +sleep 500 $ts = $ts + 1 sql insert into $tb values ( $ts , -1, -1, -1, -1, -1) @@ -47,7 +47,7 @@ $ts = $ts0 + $delta $ts = $ts + 1 sql import into $tb values ( $ts , -2, -2, -2, -2, -2) -sleep 3000 +sleep 500 sql show databases diff --git a/tests/script/general/parser/import_file.sim b/tests/script/general/parser/import_file.sim index 6b4dd07c7981ca141765b41cca3d005874a6583f..e50fc92e28ee498989f8b71d1bc5dd50faa7baa3 100644 --- a/tests/script/general/parser/import_file.sim +++ b/tests/script/general/parser/import_file.sim @@ -3,9 +3,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 2000 +sleep 500 sql connect -sleep 2000 +sleep 500 sql drop database if exists indb diff --git a/tests/script/general/parser/insert_multiTbl.sim b/tests/script/general/parser/insert_multiTbl.sim index 887f97a198237ff9c181c3d313525ae4fb66aec5..e9ee4fcf98666ddf81816c367927c8e528de3f42 100644 --- a/tests/script/general/parser/insert_multiTbl.sim +++ b/tests/script/general/parser/insert_multiTbl.sim @@ -4,9 +4,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 2000 -sql connect sleep 500 +sql connect +sleep 100 print ======================== dnode1 start sql create database mul_db diff --git a/tests/script/general/parser/insert_tb.sim b/tests/script/general/parser/insert_tb.sim index 0a9eb9f678c12de13b33e7dd1ca438c768addf81..f212325f26315f815a1a9f69ea84844b57579081 100644 --- a/tests/script/general/parser/insert_tb.sim +++ b/tests/script/general/parser/insert_tb.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ======================== dnode1 start @@ -103,7 +103,7 @@ if $rows != 1 then endi sql drop database $db -sleep 500 +sleep 100 sql create database $db sql use $db sql create table stb1 (ts timestamp, c1 int) tags(t1 int) @@ -136,7 +136,7 @@ if $data21 != 1.000000000 then endi sql drop database $db -sleep 500 +sleep 100 sql create database $db sql use $db sql create table stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 nchar(10), c6 binary(20)) tags(t1 int, t2 bigint, t3 double, t4 float, t5 nchar(10)) diff --git a/tests/script/general/parser/interp.sim b/tests/script/general/parser/interp.sim index 0d5c1804dd6cd4e929aaaa4de2e6775cc5b71153..13b6a08024206b99b410ac06a913d14078c11bfc 100644 --- a/tests/script/general/parser/interp.sim +++ b/tests/script/general/parser/interp.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = intp_db @@ -59,7 +59,7 @@ run general/parser/interp_test.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/interp_test.sim b/tests/script/general/parser/interp_test.sim index 42738a8bd134c3b7cc72569fe490285baa694e57..28601808306334286d2a9691626a2b1c1d211b30 100644 --- a/tests/script/general/parser/interp_test.sim +++ b/tests/script/general/parser/interp_test.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = intp_db @@ -638,209 +638,293 @@ if $data24 != NULL then return -1 endi - ## interp(*) from stb + group by + fill(prev) - $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(prev) group by tbname - print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 - print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 - if $rows != $tbNum then - return -1 - endi - if $data00 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data01 != 0 then - return -1 - endi - if $data02 != 0 then - return -1 - endi - if $data03 != 0.00000 then - return -1 - endi - if $data04 != 0.000000000 then - return -1 - endi - if $data05 != 0 then - return -1 - endi - if $data06 != 0 then - return -1 - endi - if $data07 != 1 then - return -1 - endi - if $data08 != binary0 then - return -1 - endi - if $data09 != nchar0 then - return -1 - endi - if $data20 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data21 != 0 then - return -1 - endi - if $data22 != NULL then - return -1 - endi - if $data23 != 0.00000 then - return -1 - endi - if $data24 != NULL then - return -1 - endi - if $data25 != 0 then - return -1 - endi - if $data26 != 0 then - return -1 - endi - if $data27 != 1 then - return -1 - endi - if $data28 != binary0 then - return -1 - endi - if $data29 != nchar0 then - return -1 - endi +## interp(*) from stb + group by + fill(prev) +$t = $ts0 + 1000 +sql select interp(*) from $stb where ts = $t fill(prev) group by tbname +print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 +print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 +if $rows != $tbNum then + return -1 +endi +if $data00 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data02 != 0 then + return -1 +endi +if $data03 != 0.00000 then + return -1 +endi +if $data04 != 0.000000000 then + return -1 +endi +if $data05 != 0 then + return -1 +endi +if $data06 != 0 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != binary0 then + return -1 +endi +if $data09 != nchar0 then + return -1 +endi +if $data20 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data21 != 0 then + return -1 +endi +if $data22 != NULL then + return -1 +endi +if $data23 != 0.00000 then + return -1 +endi +if $data24 != NULL then + return -1 +endi +if $data25 != 0 then + return -1 +endi +if $data26 != 0 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != binary0 then + return -1 +endi +if $data29 != nchar0 then + return -1 +endi - ## interp(*) from stb + group by + fill(linear) - $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(linear) group by tbname - print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 - print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 - if $rows != $tbNum then - return -1 - endi - if $data00 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data01 != 0 then - return -1 - endi - if $data02 != 0 then - return -1 - endi - if $data03 != 0.00167 then - return -1 - endi - if $data04 != 0.001666667 then - return -1 - endi - if $data05 != 0 then - return -1 - endi - if $data06 != 0 then - return -1 - endi - if $data07 != NULL then - return -1 - endi - if $data08 != NULL then - return -1 - endi - if $data09 != NULL then - return -1 - endi - if $data20 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data21 != 0 then - return -1 - endi - if $data22 != NULL then - return -1 - endi - if $data23 != 0.00167 then - return -1 - endi - if $data24 != NULL then - return -1 - endi - if $data25 != 0 then - return -1 - endi - if $data26 != 0 then - return -1 - endi - if $data27 != NULL then - return -1 - endi - if $data28 != NULL then - return -1 - endi - if $data29 != NULL then - return -1 - endi +## interp(*) from stb + group by + fill(linear) +$t = $ts0 + 1000 +sql select interp(*) from $stb where ts = $t fill(linear) group by tbname +print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 +print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 +if $rows != $tbNum then + return -1 +endi +if $data00 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data02 != 0 then + return -1 +endi +if $data03 != 0.00167 then + return -1 +endi +if $data04 != 0.001666667 then + return -1 +endi +if $data05 != 0 then + return -1 +endi +if $data06 != 0 then + return -1 +endi +if $data07 != NULL then + return -1 +endi +if $data08 != NULL then + return -1 +endi +if $data09 != NULL then + return -1 +endi +if $data20 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data21 != 0 then + return -1 +endi +if $data22 != NULL then + return -1 +endi +if $data23 != 0.00167 then + return -1 +endi +if $data24 != NULL then + return -1 +endi +if $data25 != 0 then + return -1 +endi +if $data26 != 0 then + return -1 +endi +if $data27 != NULL then + return -1 +endi +if $data28 != NULL then + return -1 +endi +if $data29 != NULL then + return -1 +endi - ## interp(*) from stb + group by + fill(value) - $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(value, -1, -2) group by tbname - print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 - print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 - if $rows != $tbNum then - return -1 - endi - if $data00 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data01 != -2 then - return -1 - endi - if $data02 != -2 then - return -1 - endi - if $data03 != -2.00000 then - return -1 - endi - if $data04 != -2.000000000 then - return -1 - endi - if $data05 != -2 then - return -1 - endi - if $data06 != -2 then - return -1 - endi - if $data07 != 1 then - return -1 - endi - if $data08 != NULL then - return -1 - endi - if $data09 != NULL then - return -1 - endi - if $data20 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data21 != -2 then - return -1 - endi - if $data22 != -2 then - return -1 - endi - if $data23 != -2.00000 then - return -1 - endi - if $data24 != -2.000000000 then - return -1 - endi - if $data25 != -2 then - return -1 - endi - if $data26 != -2 then - return -1 - endi - if $data27 != 1 then - return -1 - endi - if $data28 != NULL then - return -1 - endi - if $data29 != NULL then - return -1 - endi + ## interp(*) from stb + group by + fill(value) +$t = $ts0 + 1000 +sql select interp(*) from $stb where ts = $t fill(value, -1, -2) group by tbname +print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 +print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 +if $rows != $tbNum then + return -1 +endi +if $data00 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data01 != -2 then + return -1 +endi +if $data02 != -2 then + return -1 +endi +if $data03 != -2.00000 then + return -1 +endi +if $data04 != -2.000000000 then + return -1 +endi +if $data05 != -2 then + return -1 +endi +if $data06 != -2 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != NULL then + return -1 +endi +if $data09 != NULL then + return -1 +endi +if $data20 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data21 != -2 then + return -1 +endi +if $data22 != -2 then + return -1 +endi +if $data23 != -2.00000 then + return -1 +endi +if $data24 != -2.000000000 then + return -1 +endi +if $data25 != -2 then + return -1 +endi +if $data26 != -2 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != NULL then + return -1 +endi +if $data29 != NULL then + return -1 +endi + +sql_error select interp(ts,c1) from intp_tb0 where ts>'2018-11-25 19:19:00' and ts<'2018-11-25 19:19:12'; +sql select interp(ts,c1) from intp_tb0 where ts>'2018-11-25 19:19:00' and ts<'2018-11-25 19:19:12' interval(1s) fill(linear); +if $rows != 0 then + return -1 +endi + +sql select interp(c1) from intp_tb0 where ts>'2018-11-25 18:09:00' and ts<'2018-11-25 19:20:12' interval(18m); +if $rows != 1 then + return -1 +endi + +if $data00 != @18-11-25 18:30:00.000@ then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select interp(c1,c3,c4,ts) from intp_tb0 where ts>'2018-11-25 18:09:00' and ts<'2018-11-25 19:20:12' interval(18m) fill(linear) +if $rows != 5 then + return -1 +endi + +if $data00 != @18-11-25 17:54:00.000@ then + return -1 +endi + +if $data01 != 0 then + return -1 +endi + +if $data02 != 0.00000 then + return -1 +endi + +if $data03 != 0.000000000 then + return -1 +endi + +if $data04 != @18-11-25 17:54:00.000@ then + return -1 +endi + +if $data10 != @18-11-25 18:12:00.000@ then + return -1 +endi + +if $data11 != 1 then + return -1 +endi + +if $data12 != 1.20000 then + return -1 +endi + +if $data13 != 1.200000000 then + return -1 +endi + +if $data14 != @18-11-25 18:12:00.000@ then + return -1 +endi + +if $data40 != @18-11-25 19:06:00.000@ then + return -1 +endi + +if $data41 != 6 then + return -1 +endi + +if $data42 != 6.60000 then + return -1 +endi + +if $data43 != 6.600000000 then + return -1 +endi + +if $data44 != @18-11-25 19:06:00.000@ then + return -1 +endi \ No newline at end of file diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 3ee90cda352002961c131b7e014f8aa6d8155c05..56f115051cd4e558839d958a5177164b9fcf7a91 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = join_db @@ -415,6 +415,7 @@ sql select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_m $val = 100 if $rows != $val then + print $rows return -1 endi @@ -514,4 +515,4 @@ sql drop table tm2; sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; sql drop database ux1; -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 1c901dd2e32b3950865510176d7cc4383a41fe97..9d7cdfe3d7598a77ade1861fe6a96513f6c828f1 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sql connect -sleep 500 +sleep 100 $dbPrefix = join_m_db $tbPrefix = join_tb diff --git a/tests/script/general/parser/lastrow.sim b/tests/script/general/parser/lastrow.sim index cc71123a77e7e48c491d8c1450addb55e243bdd6..d1eadfb67a43c027b54c66967d8818454cc81d81 100644 --- a/tests/script/general/parser/lastrow.sim +++ b/tests/script/general/parser/lastrow.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = lr_db @@ -62,11 +62,11 @@ run general/parser/lastrow_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 run general/parser/lastrow_query.sim diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 5b9c8b60c3f62b48159d13f5cb82ccffd32d0feb..a87b3cc646871e442cd3540fa163f1d5492c1456 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = lr_db diff --git a/tests/script/general/parser/limit.sim b/tests/script/general/parser/limit.sim index 2089cd3d2af96e95a910acaf1067b90aa40c26be..17636dfb74d117187db66f5b66918fdd4ba9500b 100644 --- a/tests/script/general/parser/limit.sim +++ b/tests/script/general/parser/limit.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = lm_db @@ -66,11 +66,11 @@ run general/parser/limit_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 run general/parser/limit_tb.sim run general/parser/limit_stb.sim diff --git a/tests/script/general/parser/limit1.sim b/tests/script/general/parser/limit1.sim index 7236421ec2eb1617c7c16b5d659b9b57f300559a..2c40f0af2bfd3b38a43cefa4501d992dcbcfa661 100644 --- a/tests/script/general/parser/limit1.sim +++ b/tests/script/general/parser/limit1.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = lm1_db @@ -61,7 +61,7 @@ run general/parser/limit1_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/limit1_stb.sim b/tests/script/general/parser/limit1_stb.sim index 7d61a826aa90c8dd5b3a62de1e9688c0278f59fd..a0e3a45d2fc28002e2e64b195191b72659849a53 100644 --- a/tests/script/general/parser/limit1_stb.sim +++ b/tests/script/general/parser/limit1_stb.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = lm1_db diff --git a/tests/script/general/parser/limit1_tb.sim b/tests/script/general/parser/limit1_tb.sim index 72b63256dba36c799e308b41bebb4a53d5d64334..300af7ac7b669088094c0ba72288f42d34ca374d 100644 --- a/tests/script/general/parser/limit1_tb.sim +++ b/tests/script/general/parser/limit1_tb.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = lm1_db diff --git a/tests/script/general/parser/limit1_tblocks100.sim b/tests/script/general/parser/limit1_tblocks100.sim index 9a123e645ce070f5146d2507b90e16b06d258c4d..45ead58ba067be400f60c39a15cc58adb2b75a05 100644 --- a/tests/script/general/parser/limit1_tblocks100.sim +++ b/tests/script/general/parser/limit1_tblocks100.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = lm1_db @@ -61,7 +61,7 @@ run general/parser/limit1_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/limit2.sim b/tests/script/general/parser/limit2.sim index 47c3eb6d08ca1f01ded0b8e4597f91d0cd2f1933..0e7e13b6de0ea5eb40fd0609f9591b839157f5d6 100644 --- a/tests/script/general/parser/limit2.sim +++ b/tests/script/general/parser/limit2.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c rowsInFileBlock -v 255 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = lm2_db @@ -69,7 +69,7 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/limit2_query.sim b/tests/script/general/parser/limit2_query.sim index f9a1dd8e4b51107d6713d52f850f792f339b3fa4..9fe287960d22642bbea0139246d3f90537fef628 100644 --- a/tests/script/general/parser/limit2_query.sim +++ b/tests/script/general/parser/limit2_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = lm2_db @@ -143,6 +143,97 @@ if $data11 != -1 then return -1 endi +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 8200 +if $rows != 8200 then + return -1 +endi + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 10 offset 8190; +if $rows != 10 then + return -1 +endi + +if $data00 != @18-10-15 19:30:00.000@ then + return -1 +endi + +if $data01 != 5 then + return -1 +endi + +if $data10 != @18-10-15 19:35:00.000@ then + return -1 +endi + +if $data11 != -1000 then + return -1 +endi + +if $data20 != @18-10-15 19:40:00.000@ then + return -1 +endi + +if $data21 != 6 then + return -1 +endi + +if $data30 != @18-10-15 19:45:00.000@ then + return -1 +endi + +if $data31 != -1000 then + return -1 +endi + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 10 offset 10001; +if $rows != 10 then + return -1 +endi + +if $data00 != @18-10-22 02:25:00.000@ then + return -1 +endi + +if $data01 != -1000 then + return -1 +endi + +if $data10 != @18-10-22 02:30:00.000@ then + return -1 +endi + +if $data11 != 1 then + return -1 +endi + +if $data20 != @18-10-22 02:35:00.000@ then + return -1 +endi + +if $data21 != -1000 then + return -1 +endi + +if $data30 != @18-10-22 02:40:00.000@ then + return -1 +endi + +if $data31 != 2 then + return -1 +endi + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 10000 offset 10001; +print ====> needs to validate the last row result +if $rows != 9998 then + return -1 +endi + + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 100 offset 20001; +if $rows != 0 then + return -1 +endi + # tb + interval + fill(linear) + limit offset $limit = $rowNum $offset = $limit / 2 diff --git a/tests/script/general/parser/limit2_tblocks100.sim b/tests/script/general/parser/limit2_tblocks100.sim index 1aaa8e885a21b4db79c28645029a015d12a24fb5..19cea74e708beadcd1c9185b8b7d911f3e4f872b 100644 --- a/tests/script/general/parser/limit2_tblocks100.sim +++ b/tests/script/general/parser/limit2_tblocks100.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c rowsInFileBlock -v 255 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = lm2_db @@ -69,7 +69,7 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 +sleep 100 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/limit_stb.sim b/tests/script/general/parser/limit_stb.sim index d929810817927572eb7e1929ce5d98cde0940916..ec7c0e0f138e677c7da95c20af4bd13908aa1a0c 100644 --- a/tests/script/general/parser/limit_stb.sim +++ b/tests/script/general/parser/limit_stb.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = lm_db diff --git a/tests/script/general/parser/limit_tb.sim b/tests/script/general/parser/limit_tb.sim index 45f55412086c2155a2c7c924603c60d5081a643f..0c987d88c99c898ef57e19c2f9cabee4b712e12d 100644 --- a/tests/script/general/parser/limit_tb.sim +++ b/tests/script/general/parser/limit_tb.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = lm_db diff --git a/tests/script/general/parser/mixed_blocks.sim b/tests/script/general/parser/mixed_blocks.sim index d3558560df95d02256e0bc6b4a04ae9fae6e5890..8208963858721c88262af385fe4d9336012d63e8 100644 --- a/tests/script/general/parser/mixed_blocks.sim +++ b/tests/script/general/parser/mixed_blocks.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = mb_db @@ -59,7 +59,7 @@ sql show databases print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 2000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed @@ -144,4 +144,20 @@ if $data03 != 319 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print ===================> TD-2488 +sql create table m1(ts timestamp, k int) tags(a int); +sql create table t1 using m1 tags(1); +sql create table t2 using m1 tags(2); +sql insert into t1 values('2020-1-1 1:1:1', 1); +sql insert into t1 values('2020-1-1 1:10:1', 2); +sql insert into t2 values('2020-1-1 1:5:1', 99); + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 500 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql select ts from m1 where ts='2020-1-1 1:5:1' +if $rows != 1 then + return -1 +endi \ No newline at end of file diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index ab4ed2607a062ed32ebe52527d30705aa135aa86..3dcfca7503cdf82b8896a715e1393e614a498c34 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ======================== dnode1 start diff --git a/tests/script/general/parser/null_char.sim b/tests/script/general/parser/null_char.sim index 7b1c81a2958c36acbe96641494a8c156dabb275d..4e7c8ab3548a54a465801c602c0d6572b347b284 100644 --- a/tests/script/general/parser/null_char.sim +++ b/tests/script/general/parser/null_char.sim @@ -3,10 +3,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect print ========== NULL_char.sim @@ -251,7 +250,7 @@ endi ################### nchar sql alter table st41 set tag tag_nchar = "��˼����" sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41 -#sleep 500 +#sleep 100 #if $data01 != ��˼���� then # print ==== expect ��˼����, actually $data01 # return -1 @@ -451,11 +450,8 @@ sql select tag_bigint, tag_smallint, tag_tinyint from st51 if $data00 != 9223372036854775807 then return -1 endi -sql alter table st51 set tag tag_bigint = 9223372036854775808 -sql select tag_bigint, tag_smallint, tag_tinyint from st51 -if $data00 != 9223372036854775807 then - return -1 -endi +sql_error alter table st51 set tag tag_bigint = 9223372036854775808 + sql alter table st51 set tag tag_bigint = -9223372036854775807 sql select tag_bigint, tag_smallint, tag_tinyint from st51 if $data00 != -9223372036854775807 then diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim index bc22bd6da9194d9798f4744dea84d999beaf6a43..a92493b7f4a9dfa180836360c7c39b37a4991c83 100644 --- a/tests/script/general/parser/projection_limit_offset.sim +++ b/tests/script/general/parser/projection_limit_offset.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = group_db @@ -317,7 +317,6 @@ sql_error select count(*) from group_tb0 where ts in ('2016-1-1 12:12:12'); sql_error select count(*) from group_tb0 where ts < '12:12:12'; #===============================sql for twa========================================== -sql_error select twa(c1) from group_tb0; sql_error select twa(c1) from group_stb0; sql_error select twa(c2) from group_stb0 where tsnow-1h group by t1; sql_error select twa(c2) from group_stb0 where tsnow-1h group by tbname,t1; @@ -335,6 +334,9 @@ sql insert into tm0 values(10000, 1) (20000, 2)(30000, 3) (40000, NULL) (50000, #=============================tbase-1205 sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL); +if $rows != 0 then + return -1 +endi print ===================>TD-1834 sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc @@ -393,7 +395,7 @@ endi #error sql sql_error select * from 1; -sql_error select 1; +#sql_error select 1; // equals to select server_status(); sql_error select k+k; sql_error select k+1; sql_error select abc(); @@ -410,7 +412,7 @@ sql_error select k, sum(k)+1 from tm0; print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/repeatStream.sim b/tests/script/general/parser/repeatStream.sim deleted file mode 100644 index 616679e78bf8095e175ef91abf96c918626d6acb..0000000000000000000000000000000000000000 --- a/tests/script/general/parser/repeatStream.sim +++ /dev/null @@ -1,9 +0,0 @@ -$i = 1 -$repeats = 5 -while $i <= $repeats - print ====== repeat: $i - run general/parser/stream.sim - $i = $i + 1 -endw - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/selectResNum.sim b/tests/script/general/parser/selectResNum.sim index 464f363222b2d135b68d7e41d18ec33d71d843d2..20b502447836409ca06c6acaebd7bd55921f7404 100644 --- a/tests/script/general/parser/selectResNum.sim +++ b/tests/script/general/parser/selectResNum.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = sc_db @@ -118,12 +118,12 @@ endw print ====== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ====== server restart completed -sleep 500 +sleep 100 sql connect -sleep 500 +sleep 100 sql use $db ##### repeat test after server restart diff --git a/tests/script/general/parser/select_across_vnodes.sim b/tests/script/general/parser/select_across_vnodes.sim index 44e5576dac11d9f8ebffb9eee8f067bab86344c1..6c473a35d14cccf5cb24792547649500a9935563 100644 --- a/tests/script/general/parser/select_across_vnodes.sim +++ b/tests/script/general/parser/select_across_vnodes.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = sav_db diff --git a/tests/script/general/parser/select_distinct_tag.sim b/tests/script/general/parser/select_distinct_tag.sim new file mode 100644 index 0000000000000000000000000000000000000000..7ea9dc444fb3befc4409a0bc0c74a3710838ebaa --- /dev/null +++ b/tests/script/general/parser/select_distinct_tag.sim @@ -0,0 +1,61 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 +system sh/exec.sh -n dnode1 -s start +sleep 100 +sql connect + +$dbPrefix = sav_db +$tbPrefix = sav_tb +$stbPrefix = sav_stb +$tbNum = 20 +$rowNum = 10 +$totalNum = $tbNum * $rowNum +$ts0 = 1537146000000 +$delta = 600000 +print ========== alter.sim +$i = 0 +$db = $dbPrefix +$stb = $stbPrefix + +sql drop database if exists $db +sql create database $db +sql use $db +print ====== create tables +sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int, t2 int) + +$i = 0 +$ts = $ts0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $stb tags( $i , 0 ) + $i = $i + 1 +endw + +print ====== table created + +#### select distinct tag +sql select distinct t1 from $stb +if $rows != $tbNum then + return -1 +endi + +#### select distinct tag +sql select distinct t2 from $stb +if $rows != 1 then + print $rows + return -1 +endi + +#### unsupport sql +sql_error select distinct t1, t2 from &stb + +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/select_from_cache_disk.sim b/tests/script/general/parser/select_from_cache_disk.sim index 0fa0848144c740c6ed8c7583cbca46816d3da6bc..3d2cc0b70060ed9124205a00e3ea9922f2ed203b 100644 --- a/tests/script/general/parser/select_from_cache_disk.sim +++ b/tests/script/general/parser/select_from_cache_disk.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = scd_db @@ -35,11 +35,11 @@ sql insert into $tb values ('2018-09-17 09:00:00.030', 3) print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 sql use $db # generate some data in cache diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index dab76f60044c7b587e6c98bb256f895aeedb656e..5428f98593ae473f9df1038ba4d68a1a2ac38386 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = select_tags_db @@ -813,8 +813,6 @@ sql_error select first(ts), first(c1),tbname from select_tags_mt0; sql_error select first(ts), last(ts), tbname from select_tags_mt0; sql_error select last_row(*), first(ts), tbname, t1, t2 from select_tags_mt0; sql_error select tbname, last_row(*), t1, first(ts) from select_tags_mt0; -sql_error select first(ts), tbname from select_tags_tb0; -sql_error select last_row(*), t1 from select_tags_tb0; sql_error select count(*), tbname from select_tags_mt0; sql_error select sum(c2), tbname from select_tags_mt0; sql_error select avg(c3), tbname from select_tags_mt0; diff --git a/tests/script/general/parser/set_tag_vals.sim b/tests/script/general/parser/set_tag_vals.sim index bf29fe3902f88ae557b53068d1678a0aa1288818..92380ace8419c7756975fa4ce19482b845f804bf 100644 --- a/tests/script/general/parser/set_tag_vals.sim +++ b/tests/script/general/parser/set_tag_vals.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = db @@ -61,7 +61,7 @@ while $i < $tbNum endw print ====== tables created -sleep 2000 +sleep 500 sql show tables if $rows != $tbNum then diff --git a/tests/script/general/parser/single_row_in_tb.sim b/tests/script/general/parser/single_row_in_tb.sim index fe1edb1f738a01831ad3e926e6b1a925b0c05dcc..bc9362904163c7abf9d32f517a31e503801fcab4 100644 --- a/tests/script/general/parser/single_row_in_tb.sim +++ b/tests/script/general/parser/single_row_in_tb.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = sr_db @@ -32,7 +32,7 @@ run general/parser/single_row_in_tb_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/single_row_in_tb_query.sim b/tests/script/general/parser/single_row_in_tb_query.sim index 9e90b91220dae51bf31c9f3dde69eb1404223e88..1f9cb8b558c90323e18602005e275f067efeb345 100644 --- a/tests/script/general/parser/single_row_in_tb_query.sim +++ b/tests/script/general/parser/single_row_in_tb_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = sr_db diff --git a/tests/script/general/parser/sliding.sim b/tests/script/general/parser/sliding.sim index 4283421169d146a19a7a46eed1c1bd819f77464d..be33c905a1786b8c56d21aa973e6eb1fb8698ad6 100644 --- a/tests/script/general/parser/sliding.sim +++ b/tests/script/general/parser/sliding.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c debugFlag -v 135 system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = sliding_db diff --git a/tests/script/general/parser/slimit.sim b/tests/script/general/parser/slimit.sim index f7a23019cf2e1bcdca7540dd4acd997f24d31e69..bfb97b52618b7167434680c41d16f23786312fea 100644 --- a/tests/script/general/parser/slimit.sim +++ b/tests/script/general/parser/slimit.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = slm_db @@ -97,11 +97,11 @@ run general/parser/slimit_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 run general/parser/slimit_query.sim diff --git a/tests/script/general/parser/slimit1.sim b/tests/script/general/parser/slimit1.sim index 7a2511eb76b2a0d1289c509876375a2a6f06419c..901da4cab226bfaba705f7a0976731f017b8d941 100644 --- a/tests/script/general/parser/slimit1.sim +++ b/tests/script/general/parser/slimit1.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = slm_alt_tg_db @@ -56,11 +56,11 @@ run general/parser/slimit1_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 run general/parser/slimit1_query.sim diff --git a/tests/script/general/parser/slimit1_query.sim b/tests/script/general/parser/slimit1_query.sim index c205d456892d79f7a9dd8fa1ec877c04afac49ef..8e3a61bee6e025b39f784c5189be4c0d72c05833 100644 --- a/tests/script/general/parser/slimit1_query.sim +++ b/tests/script/general/parser/slimit1_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = slm_alt_tg_db diff --git a/tests/script/general/parser/slimit_alter_tags.sim b/tests/script/general/parser/slimit_alter_tags.sim index 1072f9ccb46c69ab1156783cbcdeb8940afb9aa4..ad557e891b0cb0b715da1c494c98e4a16213db8e 100644 --- a/tests/script/general/parser/slimit_alter_tags.sim +++ b/tests/script/general/parser/slimit_alter_tags.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = slm_alt_tg_db @@ -93,7 +93,7 @@ if $data02 != tb0 then return -1 endi -sleep 2000 +sleep 500 sql reset query cache sql select count(*), first(ts) from stb group by tg_added order by tg_added asc slimit 5 soffset 3 if $rows != 5 then @@ -171,11 +171,11 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 sql use $db ### repeat above queries diff --git a/tests/script/general/parser/slimit_query.sim b/tests/script/general/parser/slimit_query.sim index 3020f8047217dfbd9f39b10b5b74b9280faa4737..0a793f0611a5875f53ac75644f3156ad9aa6cc65 100644 --- a/tests/script/general/parser/slimit_query.sim +++ b/tests/script/general/parser/slimit_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = slm_db diff --git a/tests/script/general/parser/stableOp.sim b/tests/script/general/parser/stableOp.sim new file mode 100644 index 0000000000000000000000000000000000000000..f4b1bd4b8aa67df48024b8f9fa2f4015bbb14662 --- /dev/null +++ b/tests/script/general/parser/stableOp.sim @@ -0,0 +1,95 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$dbPrefix = fi_in_db +$tbPrefix = fi_in_tb +$stbPrefix = fi_in_stb +$mtPrefix = fi_in_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print create_tb test +print =============== set up +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i +$tb = $tbPrefix . $i +$stb = $stbPrefix . $i + +sql create database $db +sql use $db + +# case1: create stable test +print =========== stableOp.sim case1: create/alter/drop stable test +sql CREATE STABLE $stb (TS TIMESTAMP, COL1 INT) TAGS (ID INT); +sql show stables + +if $rows != 1 then + return -1 +endi +print data00 = $data00 +if $data00 != $stb then + return -1 +endi + +sql_error CREATE STABLE $tb using $stb tags (1); + +sql create table $tb using $stb tags (2); +sql show tables + +if $rows != 1 then + return -1 +endi + +sql alter stable $stb add column COL2 DOUBLE; + +sql insert into $tb values (now, 1, 2.0); + +sql select * from $tb ; + +if $rows != 1 then + return -1 +endi + +sql alter stable $stb drop column COL2; + +sql_error insert into $tb values (now, 1, 2.0); + +sql alter stable $stb add tag tag2 int; + +sql alter stable $stb change tag tag2 tag3; + +sql_error drop stable $tb + +sql drop table $tb ; + +sql show tables + +if $rows != 0 then + return -1 +endi + +sql DROP STABLE $stb +sql show stables + +if $rows != 0 then + return -1 +endi + +print create/alter/drop stable test passed + +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim deleted file mode 100644 index 1c8eb82c79a537ddbb322bdb164541263d92373c..0000000000000000000000000000000000000000 --- a/tests/script/general/parser/stream_on_sys.sim +++ /dev/null @@ -1,62 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 500 -sql connect -print ======================== stream_on_sys.sim - -$db = log -$tb = tb -$mt = mt -$strm = strm -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -$i = 0 - -sql use $db - -sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) -sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) -sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) -sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) -sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) -sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) -sleep 120000 -sql select * from cpustrm -if $rows <= 0 then - return -1 -endi - -sql select * from memstrm -if $rows <= 0 then - return -1 -endi - -sql select * from diskstrm -if $rows <= 0 then - return -1 -endi - -sql select * from bandstrm -if $rows <= 0 then - return -1 -endi - -sql select * from reqstrm -if $rows <= 0 then - return -1 -endi - -sql select * from iostrm -if $rows <= 0 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/tags_dynamically_specifiy.sim b/tests/script/general/parser/tags_dynamically_specifiy.sim index 8303a9c86d610476d6dba164e0af3e6a9199b1da..87b278da095463e2c51bb4c0c8bec8b22d2e873c 100644 --- a/tests/script/general/parser/tags_dynamically_specifiy.sim +++ b/tests/script/general/parser/tags_dynamically_specifiy.sim @@ -3,9 +3,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect -sleep 500 +sleep 100 $db = dytag_db $tbNum = 10 diff --git a/tests/script/general/parser/tags_filter.sim b/tests/script/general/parser/tags_filter.sim index c3d0fdfc61d2d9ec8aa19100511577652076d91d..f0ac3bdcbafcc06c7040215fa4d9e54948ac0758 100644 --- a/tests/script/general/parser/tags_filter.sim +++ b/tests/script/general/parser/tags_filter.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $db = tf_db @@ -73,6 +73,9 @@ endi sql select * from stb where t1 > '1' sql select * from stb where t1 > 'a' +print =====================> TD-2685 +sql_error select t1, count(t1) from stb; + ## wildcard '%' #sql select * from stb where t1 like '%' #if $rows != 1 then @@ -149,4 +152,57 @@ if $rows != 2 then return -1 endi +print ==================>td-2424 +sql create table t1(ts timestamp, k float) +sql insert into t1 values(now, 8.001) +sql select * from t1 where k=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k<8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k<=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k>8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k>=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k<>8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k>=8.001 and k<=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k>=8.0009999 and k<=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k>8.001 and k<=8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k>=8.001 and k<8.001 +if $rows != 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/tbnameIn.sim b/tests/script/general/parser/tbnameIn.sim index d0f74ae53d5c696cb598b471dfae32307dfd0f73..2ee5f38ab1b48a485be06376da08612bee9b98e8 100644 --- a/tests/script/general/parser/tbnameIn.sim +++ b/tests/script/general/parser/tbnameIn.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = ti_db @@ -67,7 +67,7 @@ run general/parser/tbnameIn_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/tbnameIn_query.sim b/tests/script/general/parser/tbnameIn_query.sim index ad7456f5578ccec76eef3262298130aaed2462c6..65bb89d549c32c8ae6d3054898ce26caf5482832 100644 --- a/tests/script/general/parser/tbnameIn_query.sim +++ b/tests/script/general/parser/tbnameIn_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = ti_db diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 8593400ce8c3d6ae5375993b0fcc1d7c36b86ca8..255389a2df12e470890ca74e9b3872a28bc119c3 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,112 +1,109 @@ run general/parser/alter.sim -sleep 500 +sleep 100 run general/parser/alter1.sim -sleep 500 +sleep 100 run general/parser/alter_stable.sim -sleep 500 +sleep 100 run general/parser/auto_create_tb.sim -sleep 500 +sleep 100 run general/parser/auto_create_tb_drop_tb.sim -sleep 500 +sleep 100 run general/parser/col_arithmetic_operation.sim -sleep 500 +sleep 100 run general/parser/columnValue.sim -sleep 500 +sleep 100 run general/parser/commit.sim -sleep 500 +sleep 100 run general/parser/create_db.sim -sleep 500 +sleep 100 run general/parser/create_mt.sim -sleep 500 +sleep 100 run general/parser/create_tb.sim -sleep 500 +sleep 100 run general/parser/dbtbnameValidate.sim -sleep 500 +sleep 100 run general/parser/fill.sim -sleep 500 +sleep 100 run general/parser/fill_stb.sim -sleep 500 +sleep 100 #run general/parser/fill_us.sim # -sleep 500 +sleep 100 run general/parser/first_last.sim -sleep 500 +sleep 100 run general/parser/import_commit1.sim -sleep 500 +sleep 100 run general/parser/import_commit2.sim -sleep 500 +sleep 100 run general/parser/import_commit3.sim -sleep 500 +sleep 100 #run general/parser/import_file.sim -sleep 500 +sleep 100 run general/parser/insert_tb.sim -sleep 500 +sleep 100 run general/parser/tags_dynamically_specifiy.sim -sleep 500 +sleep 100 run general/parser/interp.sim -sleep 500 +sleep 100 run general/parser/lastrow.sim -sleep 500 +sleep 100 run general/parser/limit.sim -sleep 500 +sleep 100 run general/parser/limit1.sim -sleep 500 +sleep 100 run general/parser/limit1_tblocks100.sim -sleep 500 +sleep 100 run general/parser/limit2.sim -sleep 500 +sleep 100 run general/parser/mixed_blocks.sim -sleep 500 +sleep 100 run general/parser/nchar.sim -sleep 500 +sleep 100 run general/parser/null_char.sim -sleep 500 +sleep 100 run general/parser/selectResNum.sim -sleep 500 +sleep 100 run general/parser/select_across_vnodes.sim -sleep 500 +sleep 100 run general/parser/select_from_cache_disk.sim -sleep 500 +sleep 100 run general/parser/set_tag_vals.sim -sleep 500 +sleep 100 run general/parser/single_row_in_tb.sim -sleep 500 +sleep 100 run general/parser/slimit.sim -sleep 500 +sleep 100 run general/parser/slimit1.sim -sleep 500 +sleep 100 run general/parser/slimit_alter_tags.sim -sleep 500 +sleep 100 run general/parser/tbnameIn.sim -sleep 500 +sleep 100 run general/parser/slimit_alter_tags.sim # persistent failed -sleep 500 +sleep 100 run general/parser/join.sim -sleep 500 +sleep 100 run general/parser/join_multivnode.sim -sleep 500 +sleep 100 run general/parser/projection_limit_offset.sim -sleep 500 +sleep 100 run general/parser/select_with_tags.sim -sleep 500 +sleep 100 run general/parser/groupby.sim -sleep 500 +sleep 100 run general/parser/tags_filter.sim -sleep 500 +sleep 100 run general/parser/topbot.sim -sleep 500 +sleep 100 run general/parser/union.sim -sleep 500 +sleep 100 run general/parser/constCol.sim -sleep 500 +sleep 100 run general/parser/where.sim -sleep 500 +sleep 100 run general/parser/timestamp.sim -sleep 500 +sleep 100 run general/parser/sliding.sim - -#sleep 500 -#run general/parser/repeatStream.sim -#sleep 500 -#run general/parser/stream_on_sys.sim -#sleep 500 -#run general/parser/stream.sim \ No newline at end of file +sleep 100 +run general/parser/function.sim +sleep 100 +run general/parser/stableOp.sim diff --git a/tests/script/general/parser/timestamp.sim b/tests/script/general/parser/timestamp.sim index 72966459d0ced737e2d0f44702cd5e6956a3aced..7d7362bcb5200f96f6b216430d5f712812071e66 100644 --- a/tests/script/general/parser/timestamp.sim +++ b/tests/script/general/parser/timestamp.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = ts_db @@ -59,10 +59,10 @@ run general/parser/timestamp_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 +sleep 100 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 run general/parser/timestamp_query.sim diff --git a/tests/script/general/parser/timestamp_query.sim b/tests/script/general/parser/timestamp_query.sim index 056e8bba6bd6587fcba2c2685d9764f3f67c42b7..4e553c73f41cc9d28025c5de08935af02ee10335 100644 --- a/tests/script/general/parser/timestamp_query.sim +++ b/tests/script/general/parser/timestamp_query.sim @@ -1,4 +1,4 @@ -sleep 500 +sleep 100 sql connect $dbPrefix = ts_db diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index 08e2f6ab00cdcdc3a974266420f7e66bb2e0a65b..f5c78d07a1d362ffdd5ebe5c989d88cc35a33e72 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = tb_db @@ -128,11 +128,11 @@ sql insert into test values(29999, 1)(70000, 2)(80000, 3) print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 100 sql select count(*) from t1.test where ts>10000 and ts<90000 interval(5000a) if $rows != 3 then @@ -169,7 +169,7 @@ endw system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start sql connect -sleep 500 +sleep 100 sql use db; $ts = 1000 @@ -221,7 +221,7 @@ sql insert into t2 values('2020-2-2 1:1:1', 1); system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start sql connect -sleep 500 +sleep 100 sql use db sql select count(*), first(ts), last(ts) from t2 interval(1d); diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim index cb46ac6b0d1a6d381d33974660b6584e12faff44..1b184245c38245a991e8d4cb16f6aab65342ac6b 100644 --- a/tests/script/general/parser/union.sim +++ b/tests/script/general/parser/union.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = union_db @@ -103,7 +103,7 @@ while $i < $tbNum endw print sleep 1sec. -sleep 500 +sleep 100 $i = 1 $tb = $tbPrefix . $i diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index c5b600b514be8047ab0292f23d69c9ddd6ef8399..5f9e0ec2083de0ee4acdd3f727189a229e6eaa38 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 500 +sleep 100 sql connect $dbPrefix = wh_db @@ -324,7 +324,7 @@ while $i < 1 endw system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 2000 +sleep 500 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/general/stable/disk.sim b/tests/script/general/stable/disk.sim index a67ef6d79097706f90c4a2eb229d5329d785e24c..35088c757c83c315b51881b4e1eaaf5f28e74529 100644 --- a/tests/script/general/stable/disk.sim +++ b/tests/script/general/stable/disk.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -58,7 +58,7 @@ endi sleep 1000 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start sleep 6000 diff --git a/tests/script/general/stable/metrics.sim b/tests/script/general/stable/metrics.sim index b94c76429dcae6dbd39d80beab03b4a5bc8b401f..5bda2ad42e48362084a5d8d8559109fc7a879d92 100644 --- a/tests/script/general/stable/metrics.sim +++ b/tests/script/general/stable/metrics.sim @@ -4,7 +4,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_me_db diff --git a/tests/script/general/stable/refcount.sim b/tests/script/general/stable/refcount.sim index 5fe95dde9fdf8bd8820b067f0fc30d01b5e89f80..e609ccc0552978a9570be5635482d269d7115af7 100644 --- a/tests/script/general/stable/refcount.sim +++ b/tests/script/general/stable/refcount.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/general/stable/show.sim b/tests/script/general/stable/show.sim index 66755edea73f30a3d3ecfc2dc090f0588dc21947..836a848ce0cfa715c6fd84a746009259f39692fb 100644 --- a/tests/script/general/stable/show.sim +++ b/tests/script/general/stable/show.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== create stable diff --git a/tests/script/general/stable/values.sim b/tests/script/general/stable/values.sim index d0c178385103f8fb719ed8d7ec35f55bd9023d22..5e5416e4e92928760f20b7509ed6d7eaad572216 100644 --- a/tests/script/general/stable/values.sim +++ b/tests/script/general/stable/values.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -18,16 +18,25 @@ sql create table vdb0.vtb00 using vdb0.mt tags( 0 ) sql create table vdb0.vtb01 using vdb0.mt tags( 0 ) sql create database vdb1 -sql create table vdb1.vtb10 using vdb0.mt tags( 1 ) -sql create table vdb1.vtb11 using vdb0.mt tags( 1 ) +sql create table vdb1.mt (ts timestamp, tbcol int) TAGS(tgcol int) +sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 ) +sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 ) +sql create table vdb1.vtb10 using vdb1.mt tags( 1 ) +sql create table vdb1.vtb11 using vdb1.mt tags( 1 ) sql create database vdb2 -sql create table vdb2.vtb20 using vdb0.mt tags( 2 ) -sql create table vdb2.vtb21 using vdb0.mt tags( 2 ) +sql create table vdb2.mt (ts timestamp, tbcol int) TAGS(tgcol int) +sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 ) +sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 ) +sql create table vdb2.vtb20 using vdb2.mt tags( 2 ) +sql create table vdb2.vtb21 using vdb2.mt tags( 2 ) sql create database vdb3 -sql create table vdb3.vtb30 using vdb0.mt tags( 3 ) -sql create table vdb3.vtb31 using vdb0.mt tags( 3 ) +sql create table vdb3.mt (ts timestamp, tbcol int) TAGS(tgcol int) +sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 ) +sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 ) +sql create table vdb3.vtb30 using vdb3.mt tags( 3 ) +sql create table vdb3.vtb31 using vdb3.mt tags( 3 ) print =============== step2 sql insert into vdb0.vtb00 values (1519833600000 , 10) (1519833600001, 20) (1519833600002, 30) @@ -40,7 +49,7 @@ sql insert into vdb3.vtb30 values (1519833600000 , 13) (1519833600001, 23) (1519 sql insert into vdb3.vtb31 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) sql select * from vdb0.mt -if $rows != 24 then +if $rows != 6 then return -1 endi @@ -55,7 +64,7 @@ sql insert into vdb3.vtb30 values (1519833600003 , 43) (1519833600005, 53) (1519 sql insert into vdb3.vtb31 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) sql select * from vdb0.mt -if $rows != 48 then +if $rows != 12 then return -1 endi @@ -66,7 +75,7 @@ sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vt sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) sql select * from vdb0.mt -if $rows != 64 then +if $rows != 16 then return -1 endi @@ -77,7 +86,7 @@ sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vt sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) sql select * from vdb0.mt -if $rows != 68 then +if $rows != 17 then return -1 endi @@ -87,7 +96,7 @@ sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.v sql select * from vdb0.mt -if $rows != 84 then +if $rows != 21 then return -1 endi @@ -97,7 +106,7 @@ sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1. sql select * from vdb0.mt -if $rows != 100 then +if $rows != 25 then return -1 endi diff --git a/tests/script/general/stable/vnode3.sim b/tests/script/general/stable/vnode3.sim index 3409aef9f88bf652979d5b16c023417c4820168f..0889a8f0b7da653687264e727e144e96b661f834 100644 --- a/tests/script/general/stable/vnode3.sim +++ b/tests/script/general/stable/vnode3.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/agg_stream.sim b/tests/script/general/stream/agg_stream.sim index 814438ac74531c565327bca6bb9615928bf289e1..65657fc33be88152040ce68c46d559e8b2c119a5 100644 --- a/tests/script/general/stream/agg_stream.sim +++ b/tests/script/general/stream/agg_stream.sim @@ -13,7 +13,7 @@ system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step2 diff --git a/tests/script/general/stream/column_stream.sim b/tests/script/general/stream/column_stream.sim index cb94904ff27903acb57b2202c790d25a9b1877ba..c43ca1fd5ac7cfc5e477c970b313414469e9a29c 100644 --- a/tests/script/general/stream/column_stream.sim +++ b/tests/script/general/stream/column_stream.sim @@ -9,7 +9,7 @@ system sh/cfg.sh -n dnode1 -c monitor -v 1 system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -121,7 +121,7 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print sleep 22 seconds sleep 22000 diff --git a/tests/script/general/stream/metrics_del.sim b/tests/script/general/stream/metrics_del.sim index e21fa5999aa1b6ce0c063af54172f222aeb983e1..030f9fc527623c67619801202042c2a604cb8778 100644 --- a/tests/script/general/stream/metrics_del.sim +++ b/tests/script/general/stream/metrics_del.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/metrics_n.sim b/tests/script/general/stream/metrics_n.sim deleted file mode 100644 index 7fc08064b29cc6f1681116be16ca460da0e41580..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/metrics_n.sim +++ /dev/null @@ -1,262 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = mn_db -$tbPrefix = mn_tb -$mtPrefix = mn_mt -$stPrefix = mn_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c3 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) -print select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) ===> $data00 $data01 $data02, $data03 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) - -print =============== step3 count32 -#total 32 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -# total 32 count in stream -$st = $stPrefix . c32 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -print =============== step4 count31 - -#total 31 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) -print ===> $data00 $data01 $data02, $data03 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -# total 31 count in stream will crash, 32 will error -$st = $stPrefix . c31 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -print =============== step5 avg ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . avg -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) - -print =============== step6 others -sql select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $mt interval(1d) -x step6 - return -1 -step6: - -$st = $stPrefix . ot -sql create table $st as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $mt interval(1d) -x step61 - return -1 -step61: - -print =============== step7 where -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 200 then - return -1 -endi - -$st = $stPrefix . wh -#sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) - -print =============== step8 as -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 200 then - return -1 -endi - -$st = $stPrefix . as -#sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7, avg(tbcol) as a8, sum(tbcol) as a9, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7 from $mt where ts < now + 4m interval(1d) - -print =============== step9 -print sleep 120 seconds -sleep 120000 - -print =============== step10 -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -$st = $stPrefix . c31 -sql select * from $st -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi -if $data04 != 200 then - return -1 -endi -if $data05 != 200 then - return -1 -endi -if $data06 != 200 then - return -1 -endi -if $data07 != 200 then - return -1 -endi -if $data08 != 200 then - return -1 -endi -if $data09 != 200 then - return -1 -endi - - - -$st = $stPrefix . avg -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - diff --git a/tests/script/general/stream/metrics_replica1_vnoden.sim b/tests/script/general/stream/metrics_replica1_vnoden.sim index dcbc8ae7de4962c8970cdffec5829c74cdfb7351..d7541bac662081277fddcc4991539119e2814e2e 100644 --- a/tests/script/general/stream/metrics_replica1_vnoden.sim +++ b/tests/script/general/stream/metrics_replica1_vnoden.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/restart_stream.sim b/tests/script/general/stream/restart_stream.sim index 138e2a6e2ef6e6aeb46c14b7e8fceb851e36eaa8..ce06f24df72d81c5b4da45cde75ff7d8bbd507d0 100644 --- a/tests/script/general/stream/restart_stream.sim +++ b/tests/script/general/stream/restart_stream.sim @@ -3,10 +3,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -98,7 +97,6 @@ print =============== step4 system sh/exec.sh -n dnode1 -s stop system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 1 system sh/exec.sh -n dnode1 -s start print =============== step5 diff --git a/tests/script/general/stream/stream_3.sim b/tests/script/general/stream/stream_3.sim index 88105a77d667e9cad499ec55cad7f0d7f8fc0c3f..632fe78c7f4ffd71da58ebc7824ee3fb5031a7c0 100644 --- a/tests/script/general/stream/stream_3.sim +++ b/tests/script/general/stream/stream_3.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/stream_restart.sim b/tests/script/general/stream/stream_restart.sim index 480b23055e90f59bd597d3319a5fe139eba83bab..dd7bdf1a914337101c94fd84f100f8bf0f0ca1a3 100644 --- a/tests/script/general/stream/stream_restart.sim +++ b/tests/script/general/stream/stream_restart.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/table_1.sim b/tests/script/general/stream/table_1.sim deleted file mode 100644 index f028b1626d156a65a3ff805f1479e9cb178a8d30..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/table_1.sim +++ /dev/null @@ -1,317 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = t1_db -$tbPrefix = t1_tb -$mtPrefix = t1_mt -$stPrefix = t1_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c1 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*) from $tb interval(1d) -print select count(*) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c1 -sql create table $st as select count(*) from $tb interval(1d) - -print =============== step3 c2 -sql select count(tbcol) from $tb interval(1d) -print select count(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c2 -sql create table $st as select count(tbcol) from $tb interval(1d) - -print =============== step4 c3 -sql select count(tbcol2) from $tb interval(1d) -print select count(tbcol2) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(tbcol2) from $tb interval(1d) - -print =============== step5 avg -sql select avg(tbcol) from $tb interval(1d) -print select avg(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 9.500000000 then - return -1 -endi - -$st = $stPrefix . av -sql create table $st as select avg(tbcol) from $tb interval(1d) - -print =============== step6 su -sql select sum(tbcol) from $tb interval(1d) -print select sum(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 190 then - return -1 -endi - -$st = $stPrefix . su -sql create table $st as select sum(tbcol) from $tb interval(1d) - -print =============== step7 mi -sql select min(tbcol) from $tb interval(1d) -print select min(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . mi -sql create table $st as select min(tbcol) from $tb interval(1d) - -print =============== step8 ma -sql select max(tbcol) from $tb interval(1d) -print select max(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . ma -sql create table $st as select max(tbcol) from $tb interval(1d) - -print =============== step9 fi -sql select first(tbcol) from $tb interval(1d) -print select first(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . fi -sql create table $st as select first(tbcol) from $tb interval(1d) - -print =============== step10 la -sql select last(tbcol) from $tb interval(1d) -print select last(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . la -sql create table $st as select last(tbcol) from $tb interval(1d) - -print =============== step11 st -sql select stddev(tbcol) from $tb interval(1d) -print select stddev(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 5.766281297 then - return -1 -endi - -$st = $stPrefix . std -sql create table $st as select stddev(tbcol) from $tb interval(1d) - -print =============== step12 le -sql select leastsquares(tbcol, 1, 1) from $tb interval(1d) -print select leastsquares(tbcol, 1, 1) from $tb interval(1d) ===> $data00 $data01 -#if $data01 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi - -$st = $stPrefix . le -sql create table $st as select leastsquares(tbcol, 1, 1) from $tb interval(1d) - -print =============== step13 -sql select top(tbcol, 1) from $tb interval(1d) - -print =============== step14 -sql select bottom(tbcol, 1) from $tb interval(1d) - -print =============== step15 pe - -sql select percentile(tbcol, 1) from $tb interval(1d) -print select percentile(tbcol, 1) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . pe -sql create table $st as select percentile(tbcol, 1) from $tb interval(1d) - -print =============== step16 -sql select diff(tbcol) from $tb interval(1d) -x step16 - return -1 -step16: - -print =============== step17 wh -sql select count(tbcol) from $tb where ts < now + 4m interval(1d) -print select count(tbcol) from $tb where ts < now + 4m interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . wh -#sql create table $st as select count(tbcol) from $tb where ts < now + 4m interval(1d) - -print =============== step18 as -sql select count(tbcol) from $tb interval(1d) -print select count(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . as -sql create table $st as select count(tbcol) as c from $tb interval(1d) - -print =============== step19 gb -sql select count(tbcol) from $tb interval(1d) group by tgcol -x step19 - return -1 -step19: - -print =============== step20 x -sql select count(tbcol) from $tb where ts < now + 4m interval(1d) group by tgcol -x step20 - return -1 -step20: - -print =============== step21 -print sleep 120 seconds -sleep 120000 - -print =============== step22 -$st = $stPrefix . c1 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c2 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . av -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 9.500000000 then - return -1 -endi - -$st = $stPrefix . su -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 190 then - return -1 -endi - -$st = $stPrefix . mi -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . ma -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . fi -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . la -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . std -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 5.766281297 then - return -1 -endi - -$st = $stPrefix . le -sql select * from $st -#print ===> select * from $st ===> $data00 $data01 -#if $data01 != @(0.000017, -25270086.331047)@ then -# return -1 -#endi - -$st = $stPrefix . pe -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0.190000000 then - return -1 -endi - -#$st = $stPrefix . wh -#sql select * from $st -#print ===> select * from $st ===> $data00 $data01 -#if $data01 != $rowNum then -# return -1 -#endi - -$st = $stPrefix . as -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi diff --git a/tests/script/general/stream/table_del.sim b/tests/script/general/stream/table_del.sim index ce4065a1a85ba2c4f8f79b77d1d82c5118091e9c..7ddce53c4f222de6a8462373322716a82d200bc2 100644 --- a/tests/script/general/stream/table_del.sim +++ b/tests/script/general/stream/table_del.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/table_n.sim b/tests/script/general/stream/table_n.sim deleted file mode 100644 index a336772a9819ec54607265913eb65c26a2ec1a28..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/table_n.sim +++ /dev/null @@ -1,293 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = tn_db -$tbPrefix = tn_tb -$mtPrefix = tn_mt -$stPrefix = tn_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c3 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) -print select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) ===> $data00 $data01 $data02, $data03 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) - -print =============== step3 count32 -#total 32 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -# total 32 count in stream -$st = $stPrefix . c32 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -print =============== step4 count31 - -#total 31 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) -print ===> $data00 $data01 $data02, $data03 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -# total 31 count in stream will crash, 32 will error -$st = $stPrefix . c31 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -print =============== step5 avg ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $tb interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . avg -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $tb interval(1d) - -print =============== step6 others -sql select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $tb interval(1d) -print ===> $data01 $data02 $data03 -if $data01 != 5.766281297 then - return -1 -endi -#if $data02 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi -if $data03 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . ot -sql create table $st as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $tb interval(1d) - -print =============== step7 total ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 5.766281297 then - return -1 -endi -if $data08 != 0.190000000 then - return -1 -endi -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . to -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb interval(1d) - -print =============== step8 where -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . wh -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) - - -print =============== step9 as -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . as -#sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, stddev(tbcol) as a7, percentile(tbcol, 1) as a8, count(tbcol) as a9, leastsquares(tbcol, 1, 1) as a10 from $tb where ts < now + 4m interval(1d) - -print =============== step10 -print sleep 120 seconds -sleep 120000 - -print =============== step11 -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . c31 -sql select * from $st -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . avg -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . ot -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 5.766281297 then - return -1 -endi -#if $data02 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi -if $data03 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . to -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 5.766281297 then - return -1 -endi -if $data08 != 0.190000000 then - return -1 -endi -if $data09 != 20 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stream/table_replica1_vnoden.sim b/tests/script/general/stream/table_replica1_vnoden.sim index 13a4a56fb300e877c01bbe52e15aa8221d246cb5..3c30897d2b76ceee358e300b048e38306f112c00 100644 --- a/tests/script/general/stream/table_replica1_vnoden.sim +++ b/tests/script/general/stream/table_replica1_vnoden.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/testSuite.sim b/tests/script/general/stream/testSuite.sim index 44f886d02b2831bbd30e7d52a381cd6ea9b7ba27..4a9912b848f1976903199ad314352e70db0ca059 100644 --- a/tests/script/general/stream/testSuite.sim +++ b/tests/script/general/stream/testSuite.sim @@ -1,8 +1,5 @@ run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/general/table/autocreate.sim b/tests/script/general/table/autocreate.sim index 8469a6484ac5aeb7e656ad63164ebfb0546af646..404c714ab4cfa85593d191fcd528f7b902e4cea9 100644 --- a/tests/script/general/table/autocreate.sim +++ b/tests/script/general/table/autocreate.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index e8d0cd7bd8bd0b6f3dd7193b12a33c70bb46f95b..c26c3c33e4102723805925cccb61c63d01c866b8 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/table/basic2.sim b/tests/script/general/table/basic2.sim index d1678e8abd984b3aaa5061feb9430550d8909feb..4286f9ee4a5fce5ae98956d8f2e4d143000cbdbf 100644 --- a/tests/script/general/table/basic2.sim +++ b/tests/script/general/table/basic2.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== one table diff --git a/tests/script/general/table/basic3.sim b/tests/script/general/table/basic3.sim index ded00e153a2be88ed8c9193d4c9051dacde1fc31..41c276ae984c73672c439f39ba7c54f94f375ef4 100644 --- a/tests/script/general/table/basic3.sim +++ b/tests/script/general/table/basic3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/table/bigint.sim b/tests/script/general/table/bigint.sim index dd61d25ef9f2c300a33ef8a30bcc17a1ab5d867d..0b2841f0f8f781cf42a27ea5db4e5ba28b59402a 100644 --- a/tests/script/general/table/bigint.sim +++ b/tests/script/general/table/bigint.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/binary.sim b/tests/script/general/table/binary.sim index cdbfcd4cbfe15438db884cd69b149c6e3519b6e6..fd2aa5ec44762a9de3c4d564ba75b7142da332c9 100644 --- a/tests/script/general/table/binary.sim +++ b/tests/script/general/table/binary.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/bool.sim b/tests/script/general/table/bool.sim index fbb6fe823cf30fc18302855aca9e847373cd1f6f..59297e90a9529856cf0996b4cbd7b2c0b2c5deb4 100644 --- a/tests/script/general/table/bool.sim +++ b/tests/script/general/table/bool.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/column2.sim b/tests/script/general/table/column2.sim index 9251e31daa633b45dae6ea7c6240d6c7a731f405..85d59fef49603400031358bd1b05cf653ab90324 100644 --- a/tests/script/general/table/column2.sim +++ b/tests/script/general/table/column2.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/general/table/column_name.sim b/tests/script/general/table/column_name.sim index 0b09f65129d2204c4088e5aff55c08384988a2b5..480bcd06107a8f6617f8380f1e496b6c13435d5e 100644 --- a/tests/script/general/table/column_name.sim +++ b/tests/script/general/table/column_name.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/column_num.sim b/tests/script/general/table/column_num.sim index d1091528b249cc35f31ab623b29d4dc3148c9836..b055027f595b4aa0358bb68678ad0ac61750b439 100644 --- a/tests/script/general/table/column_num.sim +++ b/tests/script/general/table/column_num.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/column_value.sim b/tests/script/general/table/column_value.sim index 117c288b3602d3ca61fe83591c6943b79903e779..92ce02ade5dad14768aded9d0ab5737a71d472d0 100644 --- a/tests/script/general/table/column_value.sim +++ b/tests/script/general/table/column_value.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/createmulti.sim b/tests/script/general/table/createmulti.sim new file mode 100644 index 0000000000000000000000000000000000000000..0da1ce96a759551936eb00b00d6915b01682a64c --- /dev/null +++ b/tests/script/general/table/createmulti.sim @@ -0,0 +1,46 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect + +print =============== create database +sql create database db +sql show databases +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 + +print =============== create super table +sql create table db.st1 (ts timestamp, i int) tags (j int) +sql create table db.st2 (ts timestamp, i int, j int) tags (t1 int, t2 int, t3 int) +sql show db.stables +if $rows != 2 then + return -1 +endi + +print $data00 $data01 $data02 + +print =============== create multiple child tables +sql create table db.ct1 using db.st1 tags(1) db.ct2 using db.st1 tags(2); + +sql show db.tables +if $rows != 2 then + return -1 +endi + +sql create table db.ct3 using db.st2 tags(1, 1, 1) db.ct4 using db.st2 tags(2, 2, 2); +sql show db.tables +if $rows != 4 then + return -1 +endi + +sql create table db.ct5 using db.st1 tags(3) db.ct6 using db.st2 tags(3, 3, 3); +sql show db.tables +if $rows != 6 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/table/date.sim b/tests/script/general/table/date.sim index 2e73a217a616859781f709cfa9c7117f6e34b9ae..7dea76dbc40473d1b97c31e07b73922c85a55108 100644 --- a/tests/script/general/table/date.sim +++ b/tests/script/general/table/date.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/db.table.sim b/tests/script/general/table/db.table.sim index 50224f83b554001cd725926986d4769d36a47cb7..717f5158c49fc98189ce1547d4c3bfdb5ed1fa51 100644 --- a/tests/script/general/table/db.table.sim +++ b/tests/script/general/table/db.table.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/delete_reuse1.sim b/tests/script/general/table/delete_reuse1.sim index 17974ccf4d3c2348c51b15e6382884d73fd73bf9..26c82e201b4b7ec0de59691516730d02a3f3693e 100644 --- a/tests/script/general/table/delete_reuse1.sim +++ b/tests/script/general/table/delete_reuse1.sim @@ -22,7 +22,7 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/table/delete_reuse2.sim b/tests/script/general/table/delete_reuse2.sim index 917893b2d286208737693d10b8ec146455c8ed3c..2e3f01a3a23a3cb0243bb78e377602bb43a8390a 100644 --- a/tests/script/general/table/delete_reuse2.sim +++ b/tests/script/general/table/delete_reuse2.sim @@ -22,7 +22,7 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/table/delete_writing.sim b/tests/script/general/table/delete_writing.sim index b55d55eb8a6ef77f453f0d60152d6e1fb7a3269c..342915f0e802a784a3a9e3ae144e6ce3b719d8ed 100644 --- a/tests/script/general/table/delete_writing.sim +++ b/tests/script/general/table/delete_writing.sim @@ -21,8 +21,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect sql create database db @@ -43,7 +41,7 @@ while $x < 15 sql create table db.tb (ts timestamp, i int) - sleep 3000 + sleep 2000 $x = $x + 1 endw diff --git a/tests/script/general/table/describe.sim b/tests/script/general/table/describe.sim index ebec004f1964119d97504f8bc78a0b42941824ab..ca5cf6f6e04925f861112063bfbab18524ecae37 100644 --- a/tests/script/general/table/describe.sim +++ b/tests/script/general/table/describe.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/double.sim b/tests/script/general/table/double.sim index 10239568fe488de55961e134a8e248cb83963393..c46029e39135cebc03a30668ec7cd784287a0fcc 100644 --- a/tests/script/general/table/double.sim +++ b/tests/script/general/table/double.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/fill.sim b/tests/script/general/table/fill.sim index 333573e577b636ef780e652d1dc61d4d6701f925..fd79e09ba16f2a43f6a16a79c45a0b6a072094ee 100644 --- a/tests/script/general/table/fill.sim +++ b/tests/script/general/table/fill.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =================== step1 @@ -42,9 +42,9 @@ sql select count(*), last(ts), min(k), max(k), avg(k) from db.mt where a=0 and t print =================== step2 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 -system sh/exec.sh -n dnode1 -s start sleep 3000 +system sh/exec.sh -n dnode1 -s start +sleep 2000 print =================== step3 sql select * from db.mt diff --git a/tests/script/general/table/float.sim b/tests/script/general/table/float.sim index e4ef8a42d67569aa75dda90236861ac11c951870..bbeb56e37099b60d7774e7a969dafd8123529843 100644 --- a/tests/script/general/table/float.sim +++ b/tests/script/general/table/float.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/int.sim b/tests/script/general/table/int.sim index 81bdcda47dd14c59df6017a07dde64da87b6546f..142c8b4f04fdf05f0c1310ed0f34f5b3dd842b95 100644 --- a/tests/script/general/table/int.sim +++ b/tests/script/general/table/int.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/limit.sim b/tests/script/general/table/limit.sim index 18597f2e1cd8ebed7747fed32067ed51664aebc8..45a20f680d0d4b8f15c2a20a9abce7d3c2075ba0 100644 --- a/tests/script/general/table/limit.sim +++ b/tests/script/general/table/limit.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 129 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 8 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/table/smallint.sim b/tests/script/general/table/smallint.sim index b6f0d6948faf9323085902cfc9b463238cb3f9ca..53dfbe15bfa5c54e575cc3f129cb6f4268050903 100644 --- a/tests/script/general/table/smallint.sim +++ b/tests/script/general/table/smallint.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/table.sim b/tests/script/general/table/table.sim index 6ad1b13b1cae6cd05b3b0bd3de4c648e4aab1fde..5d6f2ee1a3db7c54c5aa3d9254a8142f22296e64 100644 --- a/tests/script/general/table/table.sim +++ b/tests/script/general/table/table.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/table/table_len.sim b/tests/script/general/table/table_len.sim index 2568ebc7a5e460284f8a4d620beffadd73e76baf..f4c27926fb1815e1e60625cf888ff4c5c6aa3cfe 100644 --- a/tests/script/general/table/table_len.sim +++ b/tests/script/general/table/table_len.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/tinyint.sim b/tests/script/general/table/tinyint.sim index 017c007e8425385c6045d8c4ea1d3f7335e62a0d..5ad8a6933a51932b127b8a64ea4a3619b25fabe4 100644 --- a/tests/script/general/table/tinyint.sim +++ b/tests/script/general/table/tinyint.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/vgroup.sim b/tests/script/general/table/vgroup.sim index f4496e2f1988f86fc756dddc124224e84ed6fc06..f7806e316e68324c157b0ba4f34ba76e347da90d 100644 --- a/tests/script/general/table/vgroup.sim +++ b/tests/script/general/table/vgroup.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/tag/3.sim b/tests/script/general/tag/3.sim index 4b3104fe1bf591b69e4cc8837df9007ed2e815a5..878fdc04142fb54b22a9a023a85b71ce6300164f 100644 --- a/tests/script/general/tag/3.sim +++ b/tests/script/general/tag/3.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/4.sim b/tests/script/general/tag/4.sim index 120ef4e4e3ba4dd5446eff7143d2dd605374fa5d..d5ac6476d9ef8a02b41465e69405b16522e9a21c 100644 --- a/tests/script/general/tag/4.sim +++ b/tests/script/general/tag/4.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql reset query cache diff --git a/tests/script/general/tag/5.sim b/tests/script/general/tag/5.sim index efe88b36503c5161179602afea18fcf6ea1a106a..ae6d49e9f8b0502d2ef177e77a60c64ea07ef35b 100644 --- a/tests/script/general/tag/5.sim +++ b/tests/script/general/tag/5.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql reset query cache diff --git a/tests/script/general/tag/6.sim b/tests/script/general/tag/6.sim index d5f0a6c4af9b13420a07d4140a456f2b5112acd2..71957dad9f7fe920615c66fddd0e8565a31753f2 100644 --- a/tests/script/general/tag/6.sim +++ b/tests/script/general/tag/6.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql reset query cache diff --git a/tests/script/general/tag/add.sim b/tests/script/general/tag/add.sim index 301ec4f825c9954f9b5790c5e27e417538c3b4e2..4a3871235ecfbd1546cbab1ba4ab90de33ac7ed8 100644 --- a/tests/script/general/tag/add.sim +++ b/tests/script/general/tag/add.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bigint.sim b/tests/script/general/tag/bigint.sim index 4fa22d3995617ec16d9ad85d5ef9a11bfe65ac06..c8e6e72825aeeb436c0e8918f437bfa42a424e3f 100644 --- a/tests/script/general/tag/bigint.sim +++ b/tests/script/general/tag/bigint.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/binary.sim b/tests/script/general/tag/binary.sim index 7c6f95b1ef785d5621d02e51e28fe96f7fe37f6c..771e7dc263a43c12db2ff3db362e7208c7469a91 100644 --- a/tests/script/general/tag/binary.sim +++ b/tests/script/general/tag/binary.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/binary_binary.sim b/tests/script/general/tag/binary_binary.sim index 077ec85b523d9fa20a90c1ec8f0e9309dd2b3ea2..5660b1b320f5a4bf95a3d39cc5dcee70db0f8885 100644 --- a/tests/script/general/tag/binary_binary.sim +++ b/tests/script/general/tag/binary_binary.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bool.sim b/tests/script/general/tag/bool.sim index 349cb738bffbb216305af0718611987ae7f20f9c..828e89f3c7c9144ce444b974042079c173eb10a5 100644 --- a/tests/script/general/tag/bool.sim +++ b/tests/script/general/tag/bool.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bool_binary.sim b/tests/script/general/tag/bool_binary.sim index a3f99e25ad8f62562cc70f3776e99397a815ae2f..fc25399c5a6e41e9c56ab93c37b6aca4af38baf1 100644 --- a/tests/script/general/tag/bool_binary.sim +++ b/tests/script/general/tag/bool_binary.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bool_int.sim b/tests/script/general/tag/bool_int.sim index 1a2726dbaa684e2c39a37f1ef29827b480fc3617..aa443cca620ca9ba032e5df15f5e52d1ee014cab 100644 --- a/tests/script/general/tag/bool_int.sim +++ b/tests/script/general/tag/bool_int.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/change.sim b/tests/script/general/tag/change.sim index a7b8554bd96f6e8857d9b6a6b2de4faa4459bf2e..6f294c0f48fb18d232a5e78f9fbec084e8dc699d 100644 --- a/tests/script/general/tag/change.sim +++ b/tests/script/general/tag/change.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -175,7 +175,7 @@ sql alter table $mt change tag tgcol3 tgcol9 sql alter table $mt change tag tgcol5 tgcol10 sql alter table $mt change tag tgcol6 tgcol11 -sleep 5000 +sleep 3000 sql reset query cache print =============== step2 diff --git a/tests/script/general/tag/column.sim b/tests/script/general/tag/column.sim index 4a36e832f1bf6f373a81318e483ee02ee6bf2e01..36610d78da826f2f037a2202bd8321b78c6833bf 100644 --- a/tests/script/general/tag/column.sim +++ b/tests/script/general/tag/column.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/commit.sim b/tests/script/general/tag/commit.sim index 8b07d98afbaf343056ca94955e1b8977e1c46fe1..bcd9d7c618f68eb356813224a7b75f6e649a2e12 100644 --- a/tests/script/general/tag/commit.sim +++ b/tests/script/general/tag/commit.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -821,9 +821,9 @@ if $data07 != 12 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step1 $i = 0 diff --git a/tests/script/general/tag/create.sim b/tests/script/general/tag/create.sim index adbb14e88aa28a1b5b1d1f7cd7bdab53c6d7c104..d387b4345fc890e8d63e35b3dfc31dfdc3121ae5 100644 --- a/tests/script/general/tag/create.sim +++ b/tests/script/general/tag/create.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/delete.sim b/tests/script/general/tag/delete.sim index f3d0735b5c2c158b1bbc9725755c7349a474178a..2a0aa27bde6ef70e5354695a83dfc2d3853e1463 100644 --- a/tests/script/general/tag/delete.sim +++ b/tests/script/general/tag/delete.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -409,7 +409,7 @@ sql alter table $mt drop tag tgcol3 sql alter table $mt drop tag tgcol4 sql alter table $mt drop tag tgcol6 -sleep 5000 +sleep 3000 print =============== step2 $i = 2 diff --git a/tests/script/general/tag/double.sim b/tests/script/general/tag/double.sim index 13dcb1fc72b7c760c325a09fe0589ce0a211e23a..514c35dc47c4cd4dfec51afafda6182981159044 100644 --- a/tests/script/general/tag/double.sim +++ b/tests/script/general/tag/double.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/filter.sim b/tests/script/general/tag/filter.sim index 75a6ed00da6d3b8173b748961d42ab8f909e7ef1..7a899a7e67d9261ea2a68c5ad1ee311f03278cb9 100644 --- a/tests/script/general/tag/filter.sim +++ b/tests/script/general/tag/filter.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/float.sim b/tests/script/general/tag/float.sim index 2352bbaf2e8a2c5e27158332434d59be62e8d9d6..bfc7e0f569702423f97ac3b101ef354dd6ee5c6c 100644 --- a/tests/script/general/tag/float.sim +++ b/tests/script/general/tag/float.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/int.sim b/tests/script/general/tag/int.sim index a93df9a57afabdd59d22e1de44504db1bdbdaef4..2518aeb285b4a18a3842354979117fa9313a024d 100644 --- a/tests/script/general/tag/int.sim +++ b/tests/script/general/tag/int.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/int_binary.sim b/tests/script/general/tag/int_binary.sim index d3795006680a771190930b446c2390a5240dcb01..cb7aa16209f4b58f9e0d5c7323aad177a0f4467b 100644 --- a/tests/script/general/tag/int_binary.sim +++ b/tests/script/general/tag/int_binary.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/int_float.sim b/tests/script/general/tag/int_float.sim index 510c4f92d0810f6a675f0b0180081c0bf552584e..afd0a3644a4bc2de59050b2eccbec6cfe0a629d2 100644 --- a/tests/script/general/tag/int_float.sim +++ b/tests/script/general/tag/int_float.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/set.sim b/tests/script/general/tag/set.sim index 112695d37b0a924779812485e22ffb5680a8856e..cbc964fad7d73b9ba6fee5128e50bb4b9dfa433b 100644 --- a/tests/script/general/tag/set.sim +++ b/tests/script/general/tag/set.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/smallint.sim b/tests/script/general/tag/smallint.sim index 18b0957b1517560bca651ef9fbd4cc9325e94a22..598b397890003a3e5e07e1470e4e27ac4dbb1373 100644 --- a/tests/script/general/tag/smallint.sim +++ b/tests/script/general/tag/smallint.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/tinyint.sim b/tests/script/general/tag/tinyint.sim index f104f8df04356487999bd4232db70b9ea067dd73..3c173a66bf57475516871fc03e337e5ce65989cd 100644 --- a/tests/script/general/tag/tinyint.sim +++ b/tests/script/general/tag/tinyint.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/user/authority.sim b/tests/script/general/user/authority.sim index 71f185043b3ef8fbe0fb09d53c9d4dae372d41fa..45230e3e8af59dcac57ff71a7802ba6a637cc8d7 100644 --- a/tests/script/general/user/authority.sim +++ b/tests/script/general/user/authority.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ============= step1 sql create user read pass 'taosdata' diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim index eb543612f560fa43be7cb8a87ba36c3a344c1212..016848c06d1a5a08f8e78a2e2ea8cdc5acc68744 100644 --- a/tests/script/general/user/monitor.sim +++ b/tests/script/general/user/monitor.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c monitor -v 1 system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ========== step2 @@ -15,18 +15,18 @@ print ========== step2 # return -1 #step21: sql drop table log.dn -x step22 - return -1 +# return -1 step22: sql drop user log -x step23 - return -1 +# return -1 step23: print ========== step3 -sleep 3000 -sql select * from log.dn -if $rows == 0 then - return -1 -endi +sleep 2000 +#sql select * from log.dn +#if $rows == 0 then +# return -1 +#endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/user/pass_alter.sim b/tests/script/general/user/pass_alter.sim index 857d658db1ee7f1b91220106983ebb95c9b687d6..964e311ec0b17eb99d8cbfd745cc4510b9a94252 100644 --- a/tests/script/general/user/pass_alter.sim +++ b/tests/script/general/user/pass_alter.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/general/user/pass_len.sim b/tests/script/general/user/pass_len.sim index d5d7b3250fe18490f92a299d100990f8b0405bca..5eb200b51f9ac28b19ea095be6530dbada9be894 100644 --- a/tests/script/general/user/pass_len.sim +++ b/tests/script/general/user/pass_len.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/user/user_len.sim b/tests/script/general/user/user_len.sim index 79b72468bbf64a5f834c34979391f43c25809c0c..55e5eb19ef476691882e7c9ba589697aef8190da 100644 --- a/tests/script/general/user/user_len.sim +++ b/tests/script/general/user/user_len.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/vector/metrics_field.sim b/tests/script/general/vector/metrics_field.sim index 2a03ccea19a46771a89755791c7b696f547d296c..84c1ed37e7ea9d924e53275962f8381cf32a615c 100644 --- a/tests/script/general/vector/metrics_field.sim +++ b/tests/script/general/vector/metrics_field.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mf_db diff --git a/tests/script/general/vector/metrics_mix.sim b/tests/script/general/vector/metrics_mix.sim index 96b2f6a7a013d4bd323de7db9c39dfcf433421b2..2c7fc86f9f3ab2af95ffaad34d276dfe10a20844 100644 --- a/tests/script/general/vector/metrics_mix.sim +++ b/tests/script/general/vector/metrics_mix.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mx_db diff --git a/tests/script/general/vector/metrics_query.sim b/tests/script/general/vector/metrics_query.sim index 824aa6f22e45153b295f2b42afe9ebcf45d00642..46afe7df20ff8b71346ad2b0834b3f0fac52602e 100644 --- a/tests/script/general/vector/metrics_query.sim +++ b/tests/script/general/vector/metrics_query.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mq_db diff --git a/tests/script/general/vector/metrics_tag.sim b/tests/script/general/vector/metrics_tag.sim index e7575cc9df5e9c0c49af8ec21ed629c73cd7a0e6..be13ac764b7cac6e00753429a2f41b6025044fa0 100644 --- a/tests/script/general/vector/metrics_tag.sim +++ b/tests/script/general/vector/metrics_tag.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mtg_db diff --git a/tests/script/general/vector/metrics_time.sim b/tests/script/general/vector/metrics_time.sim index bbcabfd1b7f81dc65f8c61f4c46d09bd18d1c35b..0b82153860ed7ea6312fbc1c74ef79acecc8dc70 100644 --- a/tests/script/general/vector/metrics_time.sim +++ b/tests/script/general/vector/metrics_time.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mt_db diff --git a/tests/script/general/vector/multi.sim b/tests/script/general/vector/multi.sim index 105f210950960ac8dc75dd38e22af1fb8dd5b060..2ca9b7f48f046ea1ad659312983f2bf90d36bef4 100644 --- a/tests/script/general/vector/multi.sim +++ b/tests/script/general/vector/multi.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mu_db diff --git a/tests/script/general/vector/single.sim b/tests/script/general/vector/single.sim index 2292d6f44ae43286d619a560320c31b3085ea267..eee5364a4fcea4de6fcf354982af29c480e5571f 100644 --- a/tests/script/general/vector/single.sim +++ b/tests/script/general/vector/single.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_si_db diff --git a/tests/script/general/vector/table_field.sim b/tests/script/general/vector/table_field.sim index a9d6910f4da4c1258d7a2efdf592352e7d043349..65c50dadc2acdbe1dd59f811dec458c3d8ff239d 100644 --- a/tests/script/general/vector/table_field.sim +++ b/tests/script/general/vector/table_field.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tf_db diff --git a/tests/script/general/vector/table_mix.sim b/tests/script/general/vector/table_mix.sim index a6c4dc92da51d312c2a65561ecb8fcbf4737239a..12808fd6a617c6f75b1daac33ecfbff82eaa953d 100644 --- a/tests/script/general/vector/table_mix.sim +++ b/tests/script/general/vector/table_mix.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tm_db diff --git a/tests/script/general/vector/table_query.sim b/tests/script/general/vector/table_query.sim index 812229afc4acaf7cb8515c3cbb483bad302b8ac0..3e9d2d0b772ffd2083333f7e764c3bfdf99e10ff 100644 --- a/tests/script/general/vector/table_query.sim +++ b/tests/script/general/vector/table_query.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tq_db diff --git a/tests/script/general/vector/table_time.sim b/tests/script/general/vector/table_time.sim index a83195beba2fe0dec305098407ec872aa47e8535..552bdb2a99199119b6a4cd1605ef04d2fde39ea6 100644 --- a/tests/script/general/vector/table_time.sim +++ b/tests/script/general/vector/table_time.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tt_db diff --git a/tests/script/general/wal/kill.sim b/tests/script/general/wal/kill.sim index 7f103874a561c2bb2534996ab30d30ab0e8907a3..94a35b636e7915842787ed982590b39a93e1f408 100644 --- a/tests/script/general/wal/kill.sim +++ b/tests/script/general/wal/kill.sim @@ -13,62 +13,62 @@ sql create table t1 (ts timestamp, i int) sql insert into t1 values(now, 1); print =============== step3 -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step4 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step5 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step6 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step7 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step8 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then diff --git a/tests/script/general/wal/maxtables.sim b/tests/script/general/wal/maxtables.sim index e504c7e92e3447f7d29dbb2dc03456c3775ced2c..acd6af1d1a1dbb02a3aef35fe2d9c07cae74b529 100644 --- a/tests/script/general/wal/maxtables.sim +++ b/tests/script/general/wal/maxtables.sim @@ -32,11 +32,11 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 -sleep 3000 +sleep 2000 print =============== step4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql select * from st; if $rows != 100 then diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim index c6f7402b874115811e6422894c612c341ea5256f..3a8952391818bc02244f7455f694b6c1fe834288 100644 --- a/tests/script/general/wal/sync.sim +++ b/tests/script/general/wal/sync.sim @@ -31,7 +31,6 @@ system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032 print ============== deploy system sh/exec.sh -n dnode1 -s start -sleep 5001 sql connect sql create dnode $hostname2 @@ -43,8 +42,8 @@ print =============== step1 $x = 0 show1: $x = $x + 1 - sleep 2000 - if $x == 5 then + sleep 1000 + if $x == 10 then return -1 endi sql show mnodes -x show1 @@ -82,7 +81,7 @@ restful d1 table_rest 1591772800 30000 restful d1 table_rest 1591872800 30000 restful d1 table_rest 1591972800 30000 -sleep 1000 +sleep 100 sql select * from table_rest; print rows: $rows if $rows != 300000 then @@ -91,29 +90,51 @@ endi print =============== step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode1 -s start -x SIGINT -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi print =============== step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode2 -s start -x SIGINT -sleep 5000 +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a2 +endi print =============== step5 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then diff --git a/tests/script/issue/TD-2677.sim b/tests/script/issue/TD-2677.sim new file mode 100644 index 0000000000000000000000000000000000000000..8d2058a3851e456dd85f8d6a295ce45b7a6fd3bd --- /dev/null +++ b/tests/script/issue/TD-2677.sim @@ -0,0 +1,111 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 + +print ============== deploy + +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +$mnode1Role = $data2_1 +print mnode1Role $mnode1Role +$mnode2Role = $data2_2 +print mnode2Role $mnode2Role +$mnode3Role = $data2_3 +print mnode3Role $mnode3Role + +if $mnode1Role != master then + goto step1 +endi +if $mnode2Role != slave then + goto step1 +endi +if $mnode3Role != slave then + goto step1 +endi + +$x = 1 +show2: + +print =============== step1 +sql create database d1 replica 2 quorum 2 +sql create table d1.t1 (ts timestamp, i int) +sql_error create table d1.t1 (ts timestamp, i int) +sql insert into d1.t1 values(now, 1) +sql select * from d1.t1; +if $rows != 1 then + return -1 +endi + +print =============== step2 +sql create database d2 replica 3 quorum 2 +sql create table d2.t1 (ts timestamp, i int) +sql_error create table d2.t1 (ts timestamp, i int) +sql insert into d2.t1 values(now, 1) +sql select * from d2.t1; +if $rows != 1 then + return -1 +endi + +print =============== step3 +sql create database d4 replica 1 quorum 1 +sql_error create database d5 replica 1 quorum 2 +sql_error create database d6 replica 1 quorum 3 +sql_error create database d7 replica 1 quorum 4 +sql_error create database d8 replica 1 quorum 0 +sql create database d9 replica 2 quorum 1 +sql create database d10 replica 2 quorum 2 +sql_error create database d11 replica 2 quorum 3 +sql_error create database d12 replica 2 quorum 4 +sql_error create database d12 replica 2 quorum 0 +sql create database d13 replica 3 quorum 1 +sql create database d14 replica 3 quorum 2 +sql_error create database d15 replica 3 quorum 3 +sql_error create database d16 replica 3 quorum 4 +sql_error create database d17 replica 3 quorum 0 + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/issue/TD-2680.sim b/tests/script/issue/TD-2680.sim new file mode 100644 index 0000000000000000000000000000000000000000..631332160fe1ede876ef2b966dfb1963ac7351ca --- /dev/null +++ b/tests/script/issue/TD-2680.sim @@ -0,0 +1,202 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 + +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode3 -c walLevel -v 2 +system sh/cfg.sh -n dnode4 -c walLevel -v 2 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c role -v 1 +system sh/cfg.sh -n dnode2 -c role -v 2 +system sh/cfg.sh -n dnode3 -c role -v 2 +system sh/cfg.sh -n dnode4 -c role -v 2 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator + +print ============== step0 +system sh/exec_tarbitrator.sh -s start + +print ============== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi + +print ============== step2 +sql show dnodes +if $rows != 4 then + return -1 +endi + +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 +print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 + +if $data30 != 0 then + return -1 +endi + +if $data32 != 0 then + return -1 +endi + +if $data33 != 0 then + return -1 +endi + +if $data34 != ready then + return -1 +endi + +if $data35 != arb then + return -1 +endi + +if $data37 != - then + return -1 +endi + +print ============== step4 +system sh/exec_tarbitrator.sh -s stop + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show dnodes +if $rows != 4 then + return -1 +endi + +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 +print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 + +if $data30 != 0 then + return -1 +endi + +if $data32 != 0 then + return -1 +endi + +if $data33 != 0 then + return -1 +endi + +if $data34 != offline then + goto step4 +endi + +if $data35 != arb then + return -1 +endi + +if $data37 != - then + return -1 +endi + +print ============== step5 +system sh/exec_tarbitrator.sh -s start + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show dnodes +if $rows != 4 then + return -1 +endi + +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 +print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 + +if $data30 != 0 then + return -1 +endi + +if $data32 != 0 then + return -1 +endi + +if $data33 != 0 then + return -1 +endi + +if $data34 != ready then + goto step5 +endi + +if $data35 != arb then + return -1 +endi + +if $data37 != - then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/issue/TD-2713.sim b/tests/script/issue/TD-2713.sim new file mode 100644 index 0000000000000000000000000000000000000000..b66c55b9b95f67eb2478791e8ee45ffd503b12be --- /dev/null +++ b/tests/script/issue/TD-2713.sim @@ -0,0 +1,145 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c slaveQuery -v 1 +system sh/cfg.sh -n dnode2 -c slaveQuery -v 1 +system sh/cfg.sh -n dnode3 -c slaveQuery -v 1 +system sh/cfg.sh -n dnode4 -c slaveQuery -v 1 + +print ========= step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi + +print ========= step2 +sql create database d1 replica 3 +sql create table d1.t1 (ts timestamp, i int) +sql insert into d1.t1 values(now, 1) + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show d1.vgroups +print online vgroups: $data03 +if $data03 != 3 then + goto step2 +endi +sleep 1000 + +print ========= step3 +$i = 0 +while $i < 100 + $i = $i + 1 + sql select * from d1.t1 + print d1.t1 rows: $rows + if $rows != 1 then + return -1 + endi +endw + +print ========= step4 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +system rm -rf ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/* +system rm -rf ../../../sim/dnode3/data/vnode/vnode2/version.json + +system sh/exec.sh -n dnode1 -s start -x SIGINT +system sh/exec.sh -n dnode2 -s start -x SIGINT +system sh/exec.sh -n dnode3 -s start -x SIGINT + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + +sql show d1.vgroups +print online vgroups: $data03 +if $data03 != 3 then + goto step4 +endi +sleep 1000 + +print ========= step5 +$i = 0 +while $i < 100 + $i = $i + 1 + sql select * from d1.t1 + if $rows != 1 then + return -1 + endi + print d1.t1 rows: $rows +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 64a6c871fcf0ae610025b060105766d3b0cf3105..4f42d043d93a76e6e0435c80ee92d4b5f022f02a 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,6 +1,10 @@ cd ../../../debug; cmake .. cd ../../../debug; make +./test.sh -f issue/TD-2677.sim +./test.sh -f issue/TD-2680.sim +./test.sh -f issue/TD-2713.sim + ./test.sh -f general/alter/cached_schema_after_alter.sim ./test.sh -f general/alter/count.sim ./test.sh -f general/alter/dnode.sim @@ -154,6 +158,8 @@ cd ../../../debug; make ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim +./test.sh -f general/parser/function.sim +./test.sh -f general/parser/select_distinct_tag.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim @@ -190,6 +196,7 @@ cd ../../../debug; make ./test.sh -f general/table/table.sim ./test.sh -f general/table/tinyint.sim ./test.sh -f general/table/vgroup.sim +./test.sh -f general/table/createmulti.sim ./test.sh -f general/tag/3.sim ./test.sh -f general/tag/4.sim @@ -285,8 +292,12 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim ./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/lossdata.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim ./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim @@ -314,6 +325,7 @@ cd ../../../debug; make ./test.sh -f unique/mnode/mgmt24.sim ./test.sh -f unique/mnode/mgmt25.sim ./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt30.sim ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim @@ -325,16 +337,12 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -./test.sh -f general/parser/stream_on_sys.sim ./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim ./test.sh -f general/stream/metrics_replica1_vnoden.sim ./test.sh -f general/stream/restart_stream.sim ./test.sh -f general/stream/stream_3.sim ./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim ./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim ./test.sh -f general/stream/table_replica1_vnoden.sim ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim @@ -383,3 +391,8 @@ cd ../../../debug; make ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +./test.sh -f general/connection/test_old_data.sim +./test.sh -f unique/dnode/datatrans_3node.sim +./test.sh -f unique/dnode/datatrans_3node_2.sim + diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt index 765e7139163952c43569e596d8ca8cf27d583cf6..17245767344e50389d10bc997be33abb78c289ef 100644 --- a/tests/script/jenkins/basic_1.txt +++ b/tests/script/jenkins/basic_1.txt @@ -1,67 +1,3 @@ -./test.sh -f general/alter/cached_schema_after_alter.sim -./test.sh -f general/alter/count.sim -./test.sh -f general/alter/dnode.sim -./test.sh -f general/alter/import.sim -./test.sh -f general/alter/insert1.sim -./test.sh -f general/alter/insert2.sim -./test.sh -f general/alter/metrics.sim -./test.sh -f general/alter/table.sim - -./test.sh -f general/cache/new_metrics.sim -./test.sh -f general/cache/restart_metrics.sim -./test.sh -f general/cache/restart_table.sim - -./test.sh -f general/connection/connection.sim - -./test.sh -f general/column/commit.sim -./test.sh -f general/column/metrics.sim -./test.sh -f general/column/table.sim - -./test.sh -f general/compress/commitlog.sim -./test.sh -f general/compress/compress.sim -./test.sh -f general/compress/compress2.sim -./test.sh -f general/compress/uncompress.sim - -./test.sh -f general/compute/avg.sim -./test.sh -f general/compute/bottom.sim -./test.sh -f general/compute/count.sim -./test.sh -f general/compute/diff.sim -./test.sh -f general/compute/diff2.sim -./test.sh -f general/compute/first.sim -./test.sh -f general/compute/interval.sim -./test.sh -f general/compute/last.sim -./test.sh -f general/compute/leastsquare.sim -./test.sh -f general/compute/max.sim -./test.sh -f general/compute/min.sim -./test.sh -f general/compute/null.sim -./test.sh -f general/compute/percentile.sim -./test.sh -f general/compute/stddev.sim -./test.sh -f general/compute/sum.sim -./test.sh -f general/compute/top.sim - -./test.sh -f general/db/alter_option.sim -./test.sh -f general/db/alter_tables_d2.sim -./test.sh -f general/db/alter_tables_v1.sim -./test.sh -f general/db/alter_tables_v4.sim -./test.sh -f general/db/alter_vgroups.sim -./test.sh -f general/db/basic.sim -./test.sh -f general/db/basic1.sim -./test.sh -f general/db/basic2.sim -./test.sh -f general/db/basic3.sim -./test.sh -f general/db/basic4.sim -./test.sh -f general/db/basic5.sim -./test.sh -f general/db/delete_reuse1.sim -./test.sh -f general/db/delete_reuse2.sim -./test.sh -f general/db/delete_reusevnode.sim -./test.sh -f general/db/delete_reusevnode2.sim -./test.sh -f general/db/delete_writing1.sim -./test.sh -f general/db/delete_writing2.sim -./test.sh -f general/db/delete.sim -./test.sh -f general/db/len.sim -./test.sh -f general/db/repeat.sim -./test.sh -f general/db/tables.sim -./test.sh -f general/db/vnodes.sim - ./test.sh -f general/field/2.sim ./test.sh -f general/field/3.sim ./test.sh -f general/field/4.sim @@ -86,10 +22,7 @@ ./test.sh -f general/http/grafana_bug.sim ./test.sh -f general/http/grafana.sim -./test.sh -f general/import/basic.sim -./test.sh -f general/import/commit.sim -./test.sh -f general/import/large.sim -./test.sh -f general/import/replica1.sim + ./test.sh -f general/insert/basic.sim ./test.sh -f general/insert/insert_drop.sim @@ -150,56 +83,5 @@ ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim - -./test.sh -f general/stable/disk.sim -./test.sh -f general/stable/dnode3.sim -./test.sh -f general/stable/metrics.sim -./test.sh -f general/stable/refcount.sim -./test.sh -f general/stable/show.sim -./test.sh -f general/stable/values.sim -./test.sh -f general/stable/vnode3.sim - -./test.sh -f general/table/autocreate.sim -./test.sh -f general/table/basic1.sim -./test.sh -f general/table/basic2.sim -./test.sh -f general/table/basic3.sim -./test.sh -f general/table/bigint.sim -./test.sh -f general/table/binary.sim -./test.sh -f general/table/bool.sim -./test.sh -f general/table/column_name.sim -./test.sh -f general/table/column_num.sim -./test.sh -f general/table/column_value.sim -./test.sh -f general/table/column2.sim -./test.sh -f general/table/date.sim -./test.sh -f general/table/db.table.sim -./test.sh -f general/table/delete_reuse1.sim -./test.sh -f general/table/delete_reuse2.sim -./test.sh -f general/table/delete_writing.sim -./test.sh -f general/table/describe.sim -./test.sh -f general/table/double.sim -./test.sh -f general/table/fill.sim -./test.sh -f general/table/float.sim -./test.sh -f general/table/int.sim -./test.sh -f general/table/limit.sim -./test.sh -f general/table/smallint.sim -./test.sh -f general/table/table_len.sim -./test.sh -f general/table/table.sim -./test.sh -f general/table/tinyint.sim -./test.sh -f general/table/vgroup.sim -./test.sh -f unique/dnode/alternativeRole.sim -./test.sh -f unique/dnode/balance1.sim -./test.sh -f unique/dnode/balance2.sim -./test.sh -f unique/dnode/balance3.sim -./test.sh -f unique/dnode/balancex.sim -./test.sh -f unique/dnode/offline1.sim -./test.sh -f unique/dnode/offline2.sim -./test.sh -f unique/dnode/reason.sim -./test.sh -f unique/dnode/remove1.sim -./test.sh -f unique/dnode/remove2.sim -./test.sh -f unique/dnode/vnode_clean.sim - -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim - -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim +./test.sh -f general/db/nosuchfile.sim +./test.sh -f general/parser/function.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_2.txt b/tests/script/jenkins/basic_2.txt index 014313fafe9c7974baa7205cca41492dedee213a..5a2a6f4062e9f35f9ef77222383ecfc800ea7574 100644 --- a/tests/script/jenkins/basic_2.txt +++ b/tests/script/jenkins/basic_2.txt @@ -26,6 +26,9 @@ cd ../../../debug; make ./test.sh -f general/tag/set.sim ./test.sh -f general/tag/smallint.sim ./test.sh -f general/tag/tinyint.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim ./test.sh -f general/user/authority.sim ./test.sh -f general/user/monitor.sim @@ -69,22 +72,3 @@ cd ../../../debug; make ./test.sh -f unique/cluster/cache.sim ./test.sh -f unique/cluster/vgroup100.sim -./test.sh -f unique/column/replica3.sim - -./test.sh -f unique/db/commit.sim -./test.sh -f unique/db/delete.sim -./test.sh -f unique/db/delete_part.sim -./test.sh -f unique/db/replica_add12.sim -./test.sh -f unique/db/replica_add13.sim -./test.sh -f unique/db/replica_add23.sim -./test.sh -f unique/db/replica_reduce21.sim -./test.sh -f unique/db/replica_reduce32.sim -./test.sh -f unique/db/replica_reduce31.sim -./test.sh -f unique/db/replica_part.sim - -./test.sh -f unique/vnode/many.sim -./test.sh -f unique/vnode/replica2_basic2.sim -./test.sh -f unique/vnode/replica2_repeat.sim -./test.sh -f unique/vnode/replica3_basic.sim -./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt index 83b10a371cbc354acc079a6307fe0dbee27e3533..f53b1b763a45659ce498f863a3843d27ef4b233d 100644 --- a/tests/script/jenkins/basic_3.txt +++ b/tests/script/jenkins/basic_3.txt @@ -1,38 +1,4 @@ -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -#./test.sh -f unique/mnode/mgmt25.sim -#./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - - - -./test.sh -f general/parser/stream_on_sys.sim -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -53,7 +19,7 @@ ./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim -./test.sh -f unique/arbitrator/insert_duplicationTs.sim + ./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim ./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim ./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim @@ -79,3 +45,14 @@ ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +./test.sh -f unique/stable/balance_replica1.sim +./test.sh -f unique/stable/dnode2_stop.sim +./test.sh -f unique/stable/dnode2.sim +./test.sh -f unique/stable/dnode3.sim +./test.sh -f unique/stable/replica2_dnode4.sim +./test.sh -f unique/stable/replica2_vnode3.sim +./test.sh -f unique/stable/replica3_dnode6.sim +./test.sh -f unique/stable/replica3_vnode3.sim + + diff --git a/tests/script/jenkins/basic_4.txt b/tests/script/jenkins/basic_4.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a7d23df719f737d0c5a3c85acfd3875b87872ab --- /dev/null +++ b/tests/script/jenkins/basic_4.txt @@ -0,0 +1,46 @@ + +./test.sh -f unique/http/admin.sim +./test.sh -f unique/http/opentsdb.sim + +./test.sh -f unique/import/replica2.sim +./test.sh -f unique/import/replica3.sim + +./test.sh -f general/alter/cached_schema_after_alter.sim +./test.sh -f general/alter/count.sim +./test.sh -f general/alter/dnode.sim +./test.sh -f general/alter/import.sim +./test.sh -f general/alter/insert1.sim +./test.sh -f general/alter/insert2.sim +./test.sh -f general/alter/metrics.sim +./test.sh -f general/alter/table.sim + +./test.sh -f general/cache/new_metrics.sim +./test.sh -f general/cache/restart_metrics.sim +./test.sh -f general/cache/restart_table.sim + +./test.sh -f general/connection/connection.sim + +./test.sh -f general/column/commit.sim +./test.sh -f general/column/metrics.sim +./test.sh -f general/column/table.sim + +./test.sh -f general/compress/commitlog.sim +./test.sh -f general/compress/compress.sim +./test.sh -f general/compress/compress2.sim +./test.sh -f general/compress/uncompress.sim + +./test.sh -f general/stable/disk.sim +./test.sh -f general/stable/dnode3.sim +./test.sh -f general/stable/metrics.sim +./test.sh -f general/stable/refcount.sim +./test.sh -f general/stable/show.sim +./test.sh -f general/stable/values.sim +./test.sh -f general/stable/vnode3.sim + +./test.sh -f unique/column/replica3.sim +./test.sh -f issue/TD-2713.sim +./test.sh -f general/parser/select_distinct_tag.sim +./test.sh -f unique/mnode/mgmt30.sim +./test.sh -f issue/TD-2677.sim +./test.sh -f issue/TD-2680.sim +./test.sh -f unique/dnode/lossdata.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_5.txt b/tests/script/jenkins/basic_5.txt new file mode 100644 index 0000000000000000000000000000000000000000..f89be9499e7a672a3c72646614552a43d1537463 --- /dev/null +++ b/tests/script/jenkins/basic_5.txt @@ -0,0 +1,19 @@ +./test.sh -f unique/dnode/alternativeRole.sim +./test.sh -f unique/dnode/balance1.sim +./test.sh -f unique/dnode/balance2.sim +./test.sh -f unique/dnode/balance3.sim +./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/offline1.sim +./test.sh -f unique/dnode/offline2.sim + +./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim +./test.sh -f general/stream/restart_stream.sim +./test.sh -f general/stream/stream_3.sim +./test.sh -f general/stream/stream_restart.sim +./test.sh -f general/stream/table_del.sim +./test.sh -f general/stream/table_replica1_vnoden.sim + +./test.sh -f general/connection/test_old_data.sim +./test.sh -f unique/dnode/datatrans_3node.sim +./test.sh -f unique/dnode/datatrans_3node_2.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_6.txt b/tests/script/jenkins/basic_6.txt new file mode 100644 index 0000000000000000000000000000000000000000..9156360a9f548ba17d9b96d297e839e6b74aaa55 --- /dev/null +++ b/tests/script/jenkins/basic_6.txt @@ -0,0 +1,37 @@ +./test.sh -f unique/dnode/reason.sim +./test.sh -f unique/dnode/remove1.sim +./test.sh -f unique/dnode/remove2.sim +./test.sh -f unique/dnode/vnode_clean.sim + +./test.sh -f unique/db/commit.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete_part.sim +./test.sh -f unique/db/replica_add12.sim +./test.sh -f unique/db/replica_add13.sim +./test.sh -f unique/db/replica_add23.sim +./test.sh -f unique/db/replica_reduce21.sim +./test.sh -f unique/db/replica_reduce32.sim +./test.sh -f unique/db/replica_reduce31.sim +./test.sh -f unique/db/replica_part.sim + +./test.sh -f unique/vnode/many.sim +./test.sh -f unique/vnode/replica2_basic2.sim +./test.sh -f unique/vnode/replica2_repeat.sim +./test.sh -f unique/vnode/replica3_basic.sim +./test.sh -f unique/vnode/replica3_repeat.sim +./test.sh -f unique/vnode/replica3_vgroup.sim + +./test.sh -f unique/dnode/monitor.sim +./test.sh -f unique/dnode/monitor_bug.sim +./test.sh -f unique/dnode/simple.sim +./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim + +./test.sh -f general/import/basic.sim +./test.sh -f general/import/commit.sim +./test.sh -f general/import/large.sim +./test.sh -f general/import/replica1.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_7.txt b/tests/script/jenkins/basic_7.txt new file mode 100644 index 0000000000000000000000000000000000000000..27d7d4ff978bb6da595b213e05c351645917175d --- /dev/null +++ b/tests/script/jenkins/basic_7.txt @@ -0,0 +1,81 @@ + +./test.sh -f general/compute/avg.sim +./test.sh -f general/compute/bottom.sim +./test.sh -f general/compute/count.sim +./test.sh -f general/compute/diff.sim +./test.sh -f general/compute/diff2.sim +./test.sh -f general/compute/first.sim +./test.sh -f general/compute/interval.sim +./test.sh -f general/compute/last.sim +./test.sh -f general/compute/leastsquare.sim +./test.sh -f general/compute/max.sim +./test.sh -f general/compute/min.sim +./test.sh -f general/compute/null.sim +./test.sh -f general/compute/percentile.sim +./test.sh -f general/compute/stddev.sim +./test.sh -f general/compute/sum.sim +./test.sh -f general/compute/top.sim + +./test.sh -f general/db/alter_option.sim +./test.sh -f general/db/alter_tables_d2.sim +./test.sh -f general/db/alter_tables_v1.sim +./test.sh -f general/db/alter_tables_v4.sim +./test.sh -f general/db/alter_vgroups.sim +./test.sh -f general/db/basic.sim +./test.sh -f general/db/basic1.sim +./test.sh -f general/db/basic2.sim +./test.sh -f general/db/basic3.sim +./test.sh -f general/db/basic4.sim +./test.sh -f general/db/basic5.sim +./test.sh -f general/db/delete_reuse1.sim +./test.sh -f general/db/delete_reuse2.sim +./test.sh -f general/db/delete_reusevnode.sim +./test.sh -f general/db/delete_reusevnode2.sim +./test.sh -f general/db/delete_writing1.sim +./test.sh -f general/db/delete_writing2.sim +./test.sh -f general/db/delete.sim +./test.sh -f general/db/len.sim +./test.sh -f general/db/repeat.sim +./test.sh -f general/db/tables.sim +./test.sh -f general/db/vnodes.sim +./test.sh -f general/table/autocreate.sim +./test.sh -f general/table/basic1.sim +./test.sh -f general/table/basic2.sim +./test.sh -f general/table/basic3.sim +./test.sh -f general/table/bigint.sim +./test.sh -f general/table/binary.sim +./test.sh -f general/table/bool.sim +./test.sh -f general/table/column_name.sim +./test.sh -f general/table/column_num.sim +./test.sh -f general/table/column_value.sim +./test.sh -f general/table/column2.sim +./test.sh -f general/table/date.sim +./test.sh -f general/table/db.table.sim +./test.sh -f general/table/delete_reuse1.sim +./test.sh -f general/table/delete_reuse2.sim +./test.sh -f general/table/delete_writing.sim +./test.sh -f general/table/describe.sim +./test.sh -f general/table/double.sim +./test.sh -f general/table/fill.sim +./test.sh -f general/table/float.sim +./test.sh -f general/table/int.sim +./test.sh -f general/table/limit.sim +./test.sh -f general/table/smallint.sim +./test.sh -f general/table/table_len.sim +./test.sh -f general/table/table.sim +./test.sh -f general/table/tinyint.sim +./test.sh -f general/table/vgroup.sim +./test.sh -f general/table/createmulti.sim + +./test.sh -f unique/mnode/mgmt20.sim +./test.sh -f unique/mnode/mgmt21.sim +./test.sh -f unique/mnode/mgmt22.sim +./test.sh -f unique/mnode/mgmt23.sim +./test.sh -f unique/mnode/mgmt24.sim +./test.sh -f unique/mnode/mgmt25.sim +./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt33.sim +./test.sh -f unique/mnode/mgmt34.sim +./test.sh -f unique/mnode/mgmtr2.sim + +./test.sh -f unique/arbitrator/insert_duplicationTs.sim \ No newline at end of file diff --git a/tests/script/jenkins/simple.txt b/tests/script/jenkins/simple.txt index 2fe364bf26e0b3edf83278b085503e6ad2ac38f0..88ff4d660131bea5dee2f5ccd0dad9d81c7bed5f 100644 --- a/tests/script/jenkins/simple.txt +++ b/tests/script/jenkins/simple.txt @@ -147,9 +147,7 @@ cd ../../../debug; make ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim ./test.sh -f general/parser/bug.sim -#./test.sh -f general/parser/stream_on_sys.sim ./test.sh -f general/parser/repeatAlter.sim -#./test.sh -f general/parser/repeatStream.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index b271f5b726bd6e9e50265420ec6cfa6695b0f551..372bdc9d9d55635de6f36a0629b2f9c2aca5df07 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -10,6 +10,7 @@ cd ../../../debug; make ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim ./test.sh -f unique/cluster/cache.sim +./test.sh -f unique/cluster/vgroup100.sim ./test.sh -f unique/column/replica3.sim @@ -25,12 +26,21 @@ cd ../../../debug; make ./test.sh -f unique/db/replica_part.sim ./test.sh -f unique/dnode/alternativeRole.sim +./test.sh -f unique/dnode/monitor.sim +./test.sh -f unique/dnode/monitor_bug.sim +./test.sh -f unique/dnode/simple.sim ./test.sh -f unique/dnode/balance1.sim ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/lossdata.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim ./test.sh -f unique/dnode/vnode_clean.sim @@ -50,12 +60,14 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt20.sim ./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim ./test.sh -f unique/mnode/mgmt25.sim ./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt30.sim ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim diff --git a/tests/script/jenkins/wbasic.txt b/tests/script/jenkins/wbasic.txt new file mode 100644 index 0000000000000000000000000000000000000000..984b15fc5a4e49a5e730667782b41afb8bdb6bb8 --- /dev/null +++ b/tests/script/jenkins/wbasic.txt @@ -0,0 +1,393 @@ +wtest.bat -f issue/TD-2677.sim +wtest.bat -f issue/TD-2680.sim +wtest.bat -f issue/TD-2713.sim + +wtest.bat -f general/alter/cached_schema_after_alter.sim +wtest.bat -f general/alter/count.sim +wtest.bat -f general/alter/dnode.sim +wtest.bat -f general/alter/import.sim +wtest.bat -f general/alter/insert1.sim +wtest.bat -f general/alter/insert2.sim +wtest.bat -f general/alter/metrics.sim +wtest.bat -f general/alter/table.sim + +wtest.bat -f general/cache/new_metrics.sim +wtest.bat -f general/cache/restart_metrics.sim +wtest.bat -f general/cache/restart_table.sim + +wtest.bat -f general/connection/connection.sim + +wtest.bat -f general/column/commit.sim +wtest.bat -f general/column/metrics.sim +wtest.bat -f general/column/table.sim + +wtest.bat -f general/compress/commitlog.sim +wtest.bat -f general/compress/compress.sim +wtest.bat -f general/compress/compress2.sim +wtest.bat -f general/compress/uncompress.sim + +wtest.bat -f general/compute/avg.sim +wtest.bat -f general/compute/bottom.sim +wtest.bat -f general/compute/count.sim +wtest.bat -f general/compute/diff.sim +wtest.bat -f general/compute/diff2.sim +wtest.bat -f general/compute/first.sim +wtest.bat -f general/compute/interval.sim +wtest.bat -f general/compute/last.sim +wtest.bat -f general/compute/leastsquare.sim +wtest.bat -f general/compute/max.sim +wtest.bat -f general/compute/min.sim +wtest.bat -f general/compute/null.sim +wtest.bat -f general/compute/percentile.sim +wtest.bat -f general/compute/stddev.sim +wtest.bat -f general/compute/sum.sim +wtest.bat -f general/compute/top.sim + +wtest.bat -f general/db/alter_option.sim +wtest.bat -f general/db/alter_tables_d2.sim +wtest.bat -f general/db/alter_tables_v1.sim +wtest.bat -f general/db/alter_tables_v4.sim +wtest.bat -f general/db/alter_vgroups.sim +wtest.bat -f general/db/basic.sim +wtest.bat -f general/db/basic1.sim +wtest.bat -f general/db/basic2.sim +wtest.bat -f general/db/basic3.sim +wtest.bat -f general/db/basic4.sim +wtest.bat -f general/db/basic5.sim +wtest.bat -f general/db/delete_reuse1.sim +wtest.bat -f general/db/delete_reuse2.sim +wtest.bat -f general/db/delete_reusevnode.sim +wtest.bat -f general/db/delete_reusevnode2.sim +wtest.bat -f general/db/delete_writing1.sim +wtest.bat -f general/db/delete_writing2.sim +wtest.bat -f general/db/delete.sim +wtest.bat -f general/db/len.sim +wtest.bat -f general/db/repeat.sim +wtest.bat -f general/db/tables.sim +wtest.bat -f general/db/vnodes.sim +wtest.bat -f general/db/nosuchfile.sim + +wtest.bat -f general/field/2.sim +wtest.bat -f general/field/3.sim +wtest.bat -f general/field/4.sim +wtest.bat -f general/field/5.sim +wtest.bat -f general/field/6.sim +wtest.bat -f general/field/bigint.sim +wtest.bat -f general/field/binary.sim +wtest.bat -f general/field/bool.sim +wtest.bat -f general/field/single.sim +wtest.bat -f general/field/smallint.sim +wtest.bat -f general/field/tinyint.sim + +wtest.bat -f general/http/autocreate.sim +wtest.bat -f general/http/chunked.sim +wtest.bat -f general/http/gzip.sim +wtest.bat -f general/http/restful.sim +wtest.bat -f general/http/restful_insert.sim +wtest.bat -f general/http/restful_limit.sim +wtest.bat -f general/http/restful_full.sim +wtest.bat -f general/http/prepare.sim +wtest.bat -f general/http/telegraf.sim +wtest.bat -f general/http/grafana_bug.sim +wtest.bat -f general/http/grafana.sim + +wtest.bat -f general/import/basic.sim +wtest.bat -f general/import/commit.sim +wtest.bat -f general/import/large.sim +wtest.bat -f general/import/replica1.sim + +wtest.bat -f general/insert/basic.sim +wtest.bat -f general/insert/insert_drop.sim +wtest.bat -f general/insert/query_block1_memory.sim +wtest.bat -f general/insert/query_block2_memory.sim +wtest.bat -f general/insert/query_block1_file.sim +wtest.bat -f general/insert/query_block2_file.sim +wtest.bat -f general/insert/query_file_memory.sim +wtest.bat -f general/insert/query_multi_file.sim +wtest.bat -f general/insert/tcp.sim + +wtest.bat -f general/parser/alter.sim +wtest.bat -f general/parser/alter1.sim +wtest.bat -f general/parser/alter_stable.sim +wtest.bat -f general/parser/auto_create_tb.sim +wtest.bat -f general/parser/auto_create_tb_drop_tb.sim +wtest.bat -f general/parser/col_arithmetic_operation.sim +wtest.bat -f general/parser/columnValue.sim +wtest.bat -f general/parser/commit.sim +wtest.bat -f general/parser/create_db.sim +wtest.bat -f general/parser/create_mt.sim +wtest.bat -f general/parser/create_tb.sim +wtest.bat -f general/parser/dbtbnameValidate.sim +wtest.bat -f general/parser/import_commit1.sim +wtest.bat -f general/parser/import_commit2.sim +wtest.bat -f general/parser/import_commit3.sim +wtest.bat -f general/parser/insert_tb.sim +wtest.bat -f general/parser/first_last.sim +wtest.bat -f general/parser/lastrow.sim +wtest.bat -f general/parser/nchar.sim +wtest.bat -f general/parser/null_char.sim +wtest.bat -f general/parser/single_row_in_tb.sim +wtest.bat -f general/parser/select_from_cache_disk.sim +wtest.bat -f general/parser/mixed_blocks.sim +wtest.bat -f general/parser/selectResNum.sim +wtest.bat -f general/parser/limit.sim +wtest.bat -f general/parser/limit1.sim +wtest.bat -f general/parser/limit1_tblocks100.sim +wtest.bat -f general/parser/select_across_vnodes.sim +wtest.bat -f general/parser/slimit1.sim +wtest.bat -f general/parser/tbnameIn.sim +wtest.bat -f general/parser/projection_limit_offset.sim +wtest.bat -f general/parser/limit2.sim +wtest.bat -f general/parser/fill.sim +wtest.bat -f general/parser/fill_stb.sim +wtest.bat -f general/parser/where.sim +wtest.bat -f general/parser/slimit.sim +wtest.bat -f general/parser/select_with_tags.sim +wtest.bat -f general/parser/interp.sim +wtest.bat -f general/parser/tags_dynamically_specifiy.sim +wtest.bat -f general/parser/groupby.sim +wtest.bat -f general/parser/set_tag_vals.sim +wtest.bat -f general/parser/tags_filter.sim +wtest.bat -f general/parser/slimit_alter_tags.sim +wtest.bat -f general/parser/join.sim +wtest.bat -f general/parser/join_multivnode.sim +wtest.bat -f general/parser/binary_escapeCharacter.sim +wtest.bat -f general/parser/repeatAlter.sim +wtest.bat -f general/parser/union.sim +wtest.bat -f general/parser/topbot.sim +wtest.bat -f general/parser/function.sim + +wtest.bat -f general/stable/disk.sim +wtest.bat -f general/stable/dnode3.sim +wtest.bat -f general/stable/metrics.sim +wtest.bat -f general/stable/refcount.sim +wtest.bat -f general/stable/show.sim +wtest.bat -f general/stable/values.sim +wtest.bat -f general/stable/vnode3.sim + +wtest.bat -f general/table/autocreate.sim +wtest.bat -f general/table/basic1.sim +wtest.bat -f general/table/basic2.sim +wtest.bat -f general/table/basic3.sim +wtest.bat -f general/table/bigint.sim +wtest.bat -f general/table/binary.sim +wtest.bat -f general/table/bool.sim +wtest.bat -f general/table/column_name.sim +wtest.bat -f general/table/column_num.sim +wtest.bat -f general/table/column_value.sim +wtest.bat -f general/table/column2.sim +wtest.bat -f general/table/date.sim +wtest.bat -f general/table/db.table.sim +wtest.bat -f general/table/delete_reuse1.sim +wtest.bat -f general/table/delete_reuse2.sim +wtest.bat -f general/table/delete_writing.sim +wtest.bat -f general/table/describe.sim +wtest.bat -f general/table/double.sim +wtest.bat -f general/table/fill.sim +wtest.bat -f general/table/float.sim +wtest.bat -f general/table/int.sim +wtest.bat -f general/table/limit.sim +wtest.bat -f general/table/smallint.sim +wtest.bat -f general/table/table_len.sim +wtest.bat -f general/table/table.sim +wtest.bat -f general/table/tinyint.sim +wtest.bat -f general/table/vgroup.sim + +wtest.bat -f general/tag/3.sim +wtest.bat -f general/tag/4.sim +wtest.bat -f general/tag/5.sim +wtest.bat -f general/tag/6.sim +wtest.bat -f general/tag/add.sim +wtest.bat -f general/tag/bigint.sim +wtest.bat -f general/tag/binary_binary.sim +wtest.bat -f general/tag/binary.sim +wtest.bat -f general/tag/bool_binary.sim +wtest.bat -f general/tag/bool_int.sim +wtest.bat -f general/tag/bool.sim +wtest.bat -f general/tag/change.sim +wtest.bat -f general/tag/column.sim +wtest.bat -f general/tag/commit.sim +wtest.bat -f general/tag/create.sim +wtest.bat -f general/tag/delete.sim +wtest.bat -f general/tag/double.sim +wtest.bat -f general/tag/filter.sim +wtest.bat -f general/tag/float.sim +wtest.bat -f general/tag/int_binary.sim +wtest.bat -f general/tag/int_float.sim +wtest.bat -f general/tag/int.sim +wtest.bat -f general/tag/set.sim +wtest.bat -f general/tag/smallint.sim +wtest.bat -f general/tag/tinyint.sim + +wtest.bat -f general/user/authority.sim +wtest.bat -f general/user/monitor.sim +wtest.bat -f general/user/pass_alter.sim +wtest.bat -f general/user/pass_len.sim +wtest.bat -f general/user/user_create.sim +wtest.bat -f general/user/user_len.sim + +wtest.bat -f general/vector/metrics_field.sim +wtest.bat -f general/vector/metrics_mix.sim +wtest.bat -f general/vector/metrics_query.sim +wtest.bat -f general/vector/metrics_tag.sim +wtest.bat -f general/vector/metrics_time.sim +wtest.bat -f general/vector/multi.sim +wtest.bat -f general/vector/single.sim +wtest.bat -f general/vector/table_field.sim +wtest.bat -f general/vector/table_mix.sim +wtest.bat -f general/vector/table_query.sim +wtest.bat -f general/vector/table_time.sim + +wtest.bat -f general/wal/sync.sim +wtest.bat -f general/wal/kill.sim +wtest.bat -f general/wal/maxtables.sim + +wtest.bat -f unique/account/account_create.sim +wtest.bat -f unique/account/account_delete.sim +wtest.bat -f unique/account/account_len.sim +wtest.bat -f unique/account/authority.sim +wtest.bat -f unique/account/basic.sim +wtest.bat -f unique/account/paras.sim +wtest.bat -f unique/account/pass_alter.sim +wtest.bat -f unique/account/pass_len.sim +wtest.bat -f unique/account/usage.sim +wtest.bat -f unique/account/user_create.sim +wtest.bat -f unique/account/user_len.sim + +wtest.bat -f unique/big/balance.sim +wtest.bat -f unique/big/maxvnodes.sim +wtest.bat -f unique/big/tcp.sim + +wtest.bat -f unique/cluster/alter.sim +wtest.bat -f unique/cluster/balance1.sim +wtest.bat -f unique/cluster/balance2.sim +wtest.bat -f unique/cluster/balance3.sim +wtest.bat -f unique/cluster/cache.sim +wtest.bat -f unique/cluster/vgroup100.sim + +wtest.bat -f unique/column/replica3.sim + +wtest.bat -f unique/db/commit.sim +wtest.bat -f unique/db/delete.sim +wtest.bat -f unique/db/delete_part.sim +wtest.bat -f unique/db/replica_add12.sim +wtest.bat -f unique/db/replica_add13.sim +wtest.bat -f unique/db/replica_add23.sim +wtest.bat -f unique/db/replica_reduce21.sim +wtest.bat -f unique/db/replica_reduce32.sim +wtest.bat -f unique/db/replica_reduce31.sim +wtest.bat -f unique/db/replica_part.sim + +wtest.bat -f unique/dnode/alternativeRole.sim +wtest.bat -f unique/dnode/monitor.sim +wtest.bat -f unique/dnode/monitor_bug.sim +wtest.bat -f unique/dnode/simple.sim +wtest.bat -f unique/dnode/balance1.sim +wtest.bat -f unique/dnode/balance2.sim +wtest.bat -f unique/dnode/balance3.sim +wtest.bat -f unique/dnode/balancex.sim +wtest.bat -f unique/dnode/data1.sim +wtest.bat -f unique/dnode/m2.sim +wtest.bat -f unique/dnode/m3.sim +wtest.bat -f unique/dnode/lossdata.sim +wtest.bat -f unique/dnode/offline1.sim +wtest.bat -f unique/dnode/offline2.sim +wtest.bat -f unique/dnode/offline3.sim +wtest.bat -f unique/dnode/reason.sim +wtest.bat -f unique/dnode/remove1.sim +wtest.bat -f unique/dnode/remove2.sim +wtest.bat -f unique/dnode/vnode_clean.sim + +wtest.bat -f unique/http/admin.sim +wtest.bat -f unique/http/opentsdb.sim + +wtest.bat -f unique/import/replica2.sim +wtest.bat -f unique/import/replica3.sim + +wtest.bat -f unique/stable/balance_replica1.sim +wtest.bat -f unique/stable/dnode2_stop.sim +wtest.bat -f unique/stable/dnode2.sim +wtest.bat -f unique/stable/dnode3.sim +wtest.bat -f unique/stable/replica2_dnode4.sim +wtest.bat -f unique/stable/replica2_vnode3.sim +wtest.bat -f unique/stable/replica3_dnode6.sim +wtest.bat -f unique/stable/replica3_vnode3.sim + +wtest.bat -f unique/mnode/mgmt20.sim +wtest.bat -f unique/mnode/mgmt21.sim +wtest.bat -f unique/mnode/mgmt22.sim +wtest.bat -f unique/mnode/mgmt23.sim +wtest.bat -f unique/mnode/mgmt24.sim +wtest.bat -f unique/mnode/mgmt25.sim +wtest.bat -f unique/mnode/mgmt26.sim +wtest.bat -f unique/mnode/mgmt30.sim +wtest.bat -f unique/mnode/mgmt33.sim +wtest.bat -f unique/mnode/mgmt34.sim +wtest.bat -f unique/mnode/mgmtr2.sim + +wtest.bat -f unique/vnode/many.sim +wtest.bat -f unique/vnode/replica2_basic2.sim +wtest.bat -f unique/vnode/replica2_repeat.sim +wtest.bat -f unique/vnode/replica3_basic.sim +wtest.bat -f unique/vnode/replica3_repeat.sim +wtest.bat -f unique/vnode/replica3_vgroup.sim + +wtest.bat -f general/stream/metrics_del.sim +wtest.bat -f general/stream/metrics_replica1_vnoden.sim +wtest.bat -f general/stream/restart_stream.sim +wtest.bat -f general/stream/stream_3.sim +wtest.bat -f general/stream/stream_restart.sim +wtest.bat -f general/stream/table_del.sim +wtest.bat -f general/stream/table_replica1_vnoden.sim + +wtest.bat -f unique/arbitrator/check_cluster_cfg_para.sim +#wtest.bat -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +wtest.bat -f unique/arbitrator/dn3_mn1_full_createTableFail.sim +wtest.bat -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +# lower the priority while file corruption +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim +wtest.bat -f unique/arbitrator/dn3_mn2_killDnode.sim +wtest.bat -f unique/arbitrator/insert_duplicationTs.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropTable_online.sim +wtest.bat -f unique/arbitrator/replica_changeWithArbitrator.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_drop.sim + +wtest.bat -f unique/arbitrator/sync_replica2_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica2_dropTable.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_drop.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropTable.sim + +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +wtest.bat -f general/connection/test_old_data.sim +wtest.bat -f unique/dnode/datatrans_3node.sim +wtest.bat -f unique/dnode/datatrans_3node_2.sim + diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index ff1f9f53559c6da89ed2cdaf77c29f1859578ae6..e5e2194e87537f8496c9f6c5832d1a7d71842494 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -134,9 +134,6 @@ run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim run general/parser/repeatAlter.sim ##unsupport run general/parser/slimit_alter_tags.sim -##unsupport run general/parser/stream_on_sys.sim -run general/parser/stream.sim -#unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim run general/stable/metrics.sim @@ -214,14 +211,8 @@ run general/vector/table_mix.sim run general/vector/table_query.sim run general/vector/table_time.sim run general/stream/restart_stream.sim -run general/stream/stream_1.sim -run general/stream/stream_2.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/metrics_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/sh/cfg.bat b/tests/script/sh/cfg.bat new file mode 100644 index 0000000000000000000000000000000000000000..49ab34383a6a6392b40314c912dd2931dbaf7a36 --- /dev/null +++ b/tests/script/sh/cfg.bat @@ -0,0 +1,42 @@ +@echo off + +if %1 == -n set NODE_NAME=%2 +if %1 == -c set CONFIG_NAME=%2 +if %1 == -v set CONFIG_VALUE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -c set CONFIG_NAME=%4 +if %3 == -v set CONFIG_VALUE=%4 +if %5 == -n set NODE_NAME=%6 +if %5 == -c set CONFIG_NAME=%6 +if %5 == -v set CONFIG_VALUE=%6 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% + +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +echo %CONFIG_NAME% %CONFIG_VALUE% >> %TAOS_CFG% diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat new file mode 100644 index 0000000000000000000000000000000000000000..921f1611fb5ea80bbeb746b693d7c529c421ef27 --- /dev/null +++ b/tests/script/sh/deploy.bat @@ -0,0 +1,106 @@ +@echo off + +rem echo Executing deploy.sh + +if %1 == -n set NODE_NAME=%2 +if %1 == -i set NODE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -i set NODE=%4 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% + +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +if not exist %SIM_DIR% mkdir %SIM_DIR% +if not exist %NODE_DIR% mkdir %NODE_DIR% +if exist %CFG_DIR% rmdir /s/q %CFG_DIR% +if exist %LOG_DIR% rmdir /s/q %LOG_DIR% +if exist %DATA_DIR% rmdir /s/q %DATA_DIR% +if not exist %CFG_DIR% mkdir %CFG_DIR% +if not exist %LOG_DIR% mkdir %LOG_DIR% +if not exist %DATA_DIR% mkdir %DATA_DIR% + +if %NODE% == 1 set NODE=7100 +if %NODE% == 2 set NODE=7200 +if %NODE% == 3 set NODE=7300 +if %NODE% == 4 set NODE=7400 +if %NODE% == 5 set NODE=7500 +if %NODE% == 6 set NODE=7600 +if %NODE% == 7 set NODE=7700 +if %NODE% == 8 set NODE=7800 + +rem set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" + +echo firstEp %fqdn%:7100 > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% +echo serverPort %NODE% >> %TAOS_CFG% +echo dataDir %DATA_DIR% >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo debugFlag 0 >> %TAOS_CFG% +echo mDebugFlag 135 >> %TAOS_CFG% +echo sdbDebugFlag 135 >> %TAOS_CFG% +echo dDebugFlag 135 >> %TAOS_CFG% +echo vDebugFlag 135 >> %TAOS_CFG% +echo tsdbDebugFlag 135 >> %TAOS_CFG% +echo cDebugFlag 135 >> %TAOS_CFG% +echo jnidebugFlag 135 >> %TAOS_CFG% +echo odbcdebugFlag 135 >> %TAOS_CFG% +echo httpDebugFlag 135 >> %TAOS_CFG% +echo monDebugFlag 135 >> %TAOS_CFG% +echo mqttDebugFlag 135 >> %TAOS_CFG% +echo qdebugFlag 135 >> %TAOS_CFG% +echo rpcDebugFlag 135 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo udebugFlag 135 >> %TAOS_CFG% +echo sdebugFlag 135 >> %TAOS_CFG% +echo wdebugFlag 135 >> %TAOS_CFG% +echo cqdebugFlag 135 >> %TAOS_CFG% +echo monitor 0 >> %TAOS_CFG% +echo monitorInterval 1 >> %TAOS_CFG% +echo http 0 >> %TAOS_CFG% +echo slaveQuery 0 >> %TAOS_CFG% +echo numOfThreadsPerCore 2.0 >> %TAOS_CFG% +echo defaultPass taosdata >> %TAOS_CFG% +echo numOfLogLines 20000000 >> %TAOS_CFG% +echo mnodeEqualVnodeNum 0 >> %TAOS_CFG% +echo balanceInterval 1 >> %TAOS_CFG% +echo clog 2 >> %TAOS_CFG% +echo days 10 >> %TAOS_CFG% +echo statusInterval 1 >> %TAOS_CFG% +echo maxVgroupsPerDb 4 >> %TAOS_CFG% +echo minTablesPerVnode 4 >> %TAOS_CFG% +echo maxTablesPerVnode 1000 >> %TAOS_CFG% +echo tableIncStepPerVnode 10000 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo numOfMnodes 1 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo fsync 0 >> %TAOS_CFG% +echo telemetryReporting 0 >> %TAOS_CFG% diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index cd2f3772eb11a0c5d3a7a69fcb3d62162eadf14c..9b61a33d45f69a200b3f364f1553bdec5a3f52b5 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -132,10 +132,12 @@ echo "cqdebugFlag 143" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG +echo "slaveQuery 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "balanceInterval 1" >> $TAOS_CFG echo "clog 2" >> $TAOS_CFG #echo "cache 1" >> $TAOS_CFG echo "days 10" >> $TAOS_CFG diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat new file mode 100644 index 0000000000000000000000000000000000000000..6651c7aa8f726d5d42d52a4c8f3a9395e7a12151 --- /dev/null +++ b/tests/script/sh/exec.bat @@ -0,0 +1,48 @@ +@echo off + +rem echo Executing exec.sh + +if %1 == -n set NODE_NAME=%2 +if %1 == -s set EXEC_OPTON=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -s set EXEC_OPTON=%4 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %EXEC_OPTON% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TAOSD=%BUILD_DIR%taosd +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TAOSD: %TAOSD% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +if %EXEC_OPTON% == start ( + echo start %TAOSD% -c %CFG_DIR% + start %TAOSD% -c %CFG_DIR% +) + +if %EXEC_OPTON% == stop ( + rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE + rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 + + for /f "tokens=1 skip=1" %%A in ( + 'wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ' + ) do ( + rem echo taskkill /IM %%A + taskkill /IM %%A > NUL 2>&1 + ) +) diff --git a/tests/script/sh/move_dnode.sh b/tests/script/sh/move_dnode.sh new file mode 100755 index 0000000000000000000000000000000000000000..d6dc4bc3eb24fe094067cffcb51a7c335a512a94 --- /dev/null +++ b/tests/script/sh/move_dnode.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +echo "Executing move_dnode.sh" + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` +echo "SCRIPT_DIR: $SCRIPT_DIR" + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim + +NODE_DIR=$SIM_DIR/$NODE_NAME + +if [ -d "$SIM_DIR/$2" ];then + rm -rf $SIM_DIR/$2 +fi +mv $SIM_DIR/$1 $SIM_DIR/$2 + +if [[ $2 =~ "dnode2" ]];then + sed -i 's/serverPort 7100/serverPort 7200/g' $SIM_DIR/$2/cfg/taos.cfg + sed -i 's/dnode1/dnode2/g' $SIM_DIR/$2/cfg/taos.cfg + sed -i 's/7100/7200/g' $SIM_DIR/$2/data/dnode/dnodeEps.json +elif [[ $2 =~ "dnode4" ]];then + sed -i 's/serverPort 7100/serverPort 7400/g' $SIM_DIR/$2/cfg/taos.cfg + sed -i 's/dnode1/dnode4/g' $SIM_DIR/$2/cfg/taos.cfg + sed -i 's/7100/7400/g' $SIM_DIR/dnode2/data/dnode/dnodeEps.json + sed -i 's/7100/7400/g' $SIM_DIR/dnode3/data/dnode/dnodeEps.json + sed -i 's/7100/7400/g' $SIM_DIR/$2/data/dnode/dnodeEps.json +fi diff --git a/tests/script/sh/mv_old_data.sh b/tests/script/sh/mv_old_data.sh new file mode 100755 index 0000000000000000000000000000000000000000..606ee171f1e6b8605fd19d542e0b0dce5dfdda28 --- /dev/null +++ b/tests/script/sh/mv_old_data.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +echo "Executing mv_old_data.sh" + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` +echo "SCRIPT_DIR: $SCRIPT_DIR" + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim + +NODE_DIR=$SIM_DIR/$NODE_NAME + +rm -rf $SIM_DIR/dnode1 +rm -rf $SIM_DIR/dnode2 +rm -rf $SIM_DIR/dnode3 + +tar zxf $SCRIPT_DIR/general/connection/sim.tar.gz -C $SIM_DIR/../ +cd $SIM_DIR/../sim +fqdn=`hostname -f || hostname` +grep 'test4' -l -r ./* | xargs sed -i "s/test4/${fqdn}/g" +grep 'dataDir' -l -r ./* | xargs sed -i "s#/root/TDengine#${TAOS_DIR}#g" diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat new file mode 100644 index 0000000000000000000000000000000000000000..ab7af2ca92023745b8b712cb78e9b168a6c00598 --- /dev/null +++ b/tests/script/sh/stop_dnodes.bat @@ -0,0 +1,6 @@ +@echo off + +rem echo taskkill /F /IM taosd.exe + +wmic process where "name='taosd.exe'" call terminate > NUL 2>&1 +taskkill /F /IM taosd.exe > NUL 2>&1 \ No newline at end of file diff --git a/tests/script/test.sh b/tests/script/test.sh index a68ac4736dc8e7c0fa3b458c722ea58e4c9aec8e..1c7a7527abc90b3acc98fdf481b847258dfda489 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -113,7 +113,6 @@ echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 143" >> $TAOS_CFG echo "udebugFlag 143" >> $TAOS_CFG -echo "tablemetakeeptimer 5" >> $TAOS_CFG echo "wal 0" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim index e11140028d474b131c472d6c0ff0f9a56587e3a9..8bca76c38b627ac6a6aa9eaae07a9126a9b19057 100644 --- a/tests/script/tmp/mnodes.sim +++ b/tests/script/tmp/mnodes.sim @@ -35,7 +35,6 @@ system sh/cfg.sh -n dnode3 -c replica -v 3 print ============== deploy system sh/exec.sh -n dnode1 -s start -sleep 5001 sql connect sql create dnode $hostname2 @@ -45,13 +44,29 @@ system sh/exec.sh -n dnode3 -s start print =============== step1 $x = 0 -show1: +step1: $x = $x + 1 - sleep 2000 - if $x == 5 then - return -1 + sleep 1000 + if $x == 10 then + return -1 endi -sql show mnodes -x show1 + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes $mnode1Role = $data2_1 print mnode1Role $mnode1Role $mnode2Role = $data2_2 @@ -60,13 +75,13 @@ $mnode3Role = $data2_3 print mnode3Role $mnode3Role if $mnode1Role != master then - goto show1 + goto step1 endi if $mnode2Role != slave then - goto show1 + goto step1 endi if $mnode3Role != slave then - goto show1 + goto step1 endi $x = 1 @@ -75,12 +90,24 @@ show2: print =============== step $x sql show mnodes print $data0_1 $data2_1 -print $data0_2 $data2_2 -print $data0_3 $data2_3 +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi $x = $x + 1 -sleep 5000 +sleep 3000 if $x == 100000 then return -1 endi diff --git a/tests/script/unique/account/account_create.sim b/tests/script/unique/account/account_create.sim index 9eecceb39d106ab335b631bcfae370c2ae516490..e36de29e7c5835ddc78a9f3eab4b2b4d34634c42 100644 --- a/tests/script/unique/account/account_create.sim +++ b/tests/script/unique/account/account_create.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/unique/account/account_delete.sim b/tests/script/unique/account/account_delete.sim index 564512228eda3aa156faba4818e3bc150b69de76..d99a8b559dc6e04e4d6996e042d915671781d699 100644 --- a/tests/script/unique/account/account_delete.sim +++ b/tests/script/unique/account/account_delete.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/unique/account/account_len.sim b/tests/script/unique/account/account_len.sim index 75636be513af84ba00d9052d09193dcbf9ee2158..f8379bdf954bdde122e68585b973f4957ef15739 100644 --- a/tests/script/unique/account/account_len.sim +++ b/tests/script/unique/account/account_len.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/unique/account/authority.sim b/tests/script/unique/account/authority.sim index f88f254a79c83a95ff0e89a3e36eddb2856ad3a2..8f2408de1429a8ea34add79e335f6bf7f42ca2b0 100644 --- a/tests/script/unique/account/authority.sim +++ b/tests/script/unique/account/authority.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/unique/account/basic.sim b/tests/script/unique/account/basic.sim index 0861c9305b836eb96c62acf11a8cad21ec6212a3..00e706a4482d9fa57ed2f97a9995ce84d3667fa1 100644 --- a/tests/script/unique/account/basic.sim +++ b/tests/script/unique/account/basic.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/paras.sim b/tests/script/unique/account/paras.sim index ae511fe2b065dd23d7b1dc19464a06b3e43e1fe0..102f5b6a381e5100b35a4f0125b1318bcb8b1d76 100644 --- a/tests/script/unique/account/paras.sim +++ b/tests/script/unique/account/paras.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/pass_alter.sim b/tests/script/unique/account/pass_alter.sim index bf939acce37cb2d878f1e31ca01e11429d8d4fcb..8b857b014a292d53536c5acf2a00daa15be11239 100644 --- a/tests/script/unique/account/pass_alter.sim +++ b/tests/script/unique/account/pass_alter.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/unique/account/pass_len.sim b/tests/script/unique/account/pass_len.sim index 1dd0a616f291549cf3e431348fe028114f11836d..f4ceb76f7b8b41873217bd11ae2c3d385386b0e9 100644 --- a/tests/script/unique/account/pass_len.sim +++ b/tests/script/unique/account/pass_len.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index 7fde3652017f5b17e61a39891b1f55c5940e2061..3b9c20b159a6237f469fc1e48b5b3a3f4ca5f7b8 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start #system sh/exec.sh -n monitor -s 1 system sh/exec.sh -n monitorInterval -s 1 -sleep 3000 +sleep 2000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/user_create.sim b/tests/script/unique/account/user_create.sim index 029bda22ea5a192634c911bba1df74a266402fea..e54a380f0dbef8107de452354ea01bc58262d548 100644 --- a/tests/script/unique/account/user_create.sim +++ b/tests/script/unique/account/user_create.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/unique/account/user_len.sim b/tests/script/unique/account/user_len.sim index 91a5ebfaaba09196ff62196b23e60b59784ee1eb..b8d448f0ffc9e43cbc0f0a5a849bda215e72e790 100644 --- a/tests/script/unique/account/user_len.sim +++ b/tests/script/unique/account/user_len.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim index 1b22db9d827d3bc01c3374f21df51377bb775aa5..7bf2c2ac42a95d1ab6da3ecf50a287eea2ce46bb 100644 --- a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim +++ b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim @@ -90,7 +90,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2~7 and add into cluster @@ -115,7 +115,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 7 then +if $rows != 8 then sleep 2000 goto wait_dnode_created endi @@ -172,7 +172,7 @@ print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode_offline_overtime_dropped endi diff --git a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim index 6f0f61d2e2f6dd85523f5e9b888172f475d8cbad..dbd0e2bd87c099dca54aa33d609696cb2dc89381 100644 --- a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim +++ b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim @@ -49,7 +49,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -57,7 +57,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 $totalTableNum = 1 $sleepTimer = 3000 @@ -178,7 +178,7 @@ endi print ============== step7: restart dnode3, waiting sync end system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode3_ready: diff --git a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim index 80a050f883dd4cb1516bee18791addadb5b4567d..e15edb3f3d444b7407244b8cf3fa781ba36357d6 100644 --- a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim +++ b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim @@ -9,11 +9,11 @@ # expect: in dnode2, the files 1837 and 1839 will be removed sql connect -sleep 3000 +sleep 2000 print ============== step7: restart dnode2, waiting sync end system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 wait_dnode2_ready: sql show dnodes diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim index 97433741f53c3053e25b588d7368377e1b890117..057654abb47be64bcba726df8b689b8db4d97f58 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -39,7 +39,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table to max tables @@ -49,7 +49,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 16 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim index a6a9129090da41d83e619d6ca123c2ba75d312fd..c597c576e2994725035f83268c6b1f99392627da 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim @@ -40,7 +40,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table to max tables @@ -50,7 +50,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 16 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim index 7bb37c5ebb65a1620f6dbcec24502e9b7e635844..49e1dba06719a3375b221854ccb3a35d1bd03dfe 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 @@ -143,7 +143,7 @@ endi print ============== step5: create the middle table 5 and insert data sql create table tb5 using $stb tags( 5 ) -sleep 3000 +sleep 2000 $tsStart = 1420041620000 $i = 5 @@ -212,7 +212,7 @@ endi print ============== step8: create the first table 0 and insert data sql create table tb0 using $stb tags( 0 ) -sleep 3000 +sleep 2000 $tsStart = 1420041640000 $i = 0 @@ -277,7 +277,7 @@ endi print ============== step11: create the last table 9 and insert data sql create table tb9 using $stb tags( 9 ) -sleep 3000 +sleep 2000 $tsStart = 1420041660000 $i = 0 diff --git a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim index e11dd69598943410def7d75abb53ce2c03fd63c0..2af7cf56b7b547a707abc18f26a18fe74bd8f95e 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim @@ -49,7 +49,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data @@ -59,7 +59,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $rowNum = 10 $tblNum = 16 diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim index aaf0da8553886b8da1f6df7107a8b25bff531705..96fde9061a0648d69386cb7f5c23450dea6aad7e 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -53,7 +53,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 5000 +sleep 3000 $sleepTimer = 3000 @@ -103,7 +103,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_offline_0 endi @@ -170,7 +170,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_reready endi @@ -225,7 +225,7 @@ if $data00 != $totalRows then endi print ============== step5: stop dnode2, and remove its vnode -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep $sleepTimer @@ -237,7 +237,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_offline endi @@ -282,7 +282,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_reready endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index c647a35a5cc21c3a071b67b27330418a7b7ac57a..da76cc467b7813586c98761319a01eb1f2ed6bb6 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 @@ -105,7 +105,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -172,7 +172,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_reready endi @@ -239,7 +239,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode23_offline endi @@ -283,7 +283,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode23_reready endi @@ -359,13 +359,15 @@ endi print ============== step7: stop dnode3/dnode2, and cluster unable to provide services system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 3000 -sql_error select count(*) from $stb +sleep 2000 +sql select count(*) from $stb -x s71 +s71: print ============== step8: restart dnode2, and cluster Still unable to provide services system sh/exec.sh -n dnode2 -s start -sleep 3000 -sql_error select count(*) from $stb +sleep 2000 +sql select count(*) from $stb -x s81 +s81: print ============== step9: restart dnode3, and cluster Resume service delivery system sh/exec.sh -n dnode3 -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim index 986df81bf695affa71a4e9be740fc246fcc0ee96..c924fee1ae673f4e5696d361be331527d41ff21d 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim @@ -70,7 +70,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster, then create database replica 2, create table , and insert data @@ -78,7 +78,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 $rowNum = 100 $tblNum = 16 @@ -139,7 +139,7 @@ if $data00 != $totalRows then return -1 endi -#sleep 3000 +#sleep 2000 #sql show dnodes #print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 #print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 @@ -175,7 +175,7 @@ endi sql show dnodes if $rows != 3 then - sleep 3000 + sleep 2000 goto wait_drop endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 @@ -218,7 +218,7 @@ system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 16 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode3_ready: @@ -230,7 +230,7 @@ endi sql show dnodes print rows: $rows if $rows != 4 then - sleep 3000 + sleep 2000 goto wait_dnode3_ready endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 @@ -271,7 +271,7 @@ sql drop database $db sleep 1000 system sh/exec.sh -n dnode5 -s start sql create dnode $hostname5 -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode5: $loopCnt = $loopCnt + 1 @@ -281,7 +281,7 @@ endi sql show dnodes if $rows != 5 then - sleep 3000 + sleep 2000 goto wait_dnode5 endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 @@ -352,7 +352,7 @@ endw print info: select count(*) from $stb sleep 2000 sql reset query cache -sleep 3000 +sleep 2000 sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 23ace47e3cf13cb18467c0482a30be97d5f03d33..b9008e4c42b4f4a34b99bb48d4e38c780e747bb4 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -45,15 +45,30 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ============== step2: start dnode2 and add into cluster , then create database with replica 1, and create table, insert data system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 -$sleepTimer = 10000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi $db = db sql create database $db replica 1 @@ -84,7 +99,6 @@ while $i < $tblNum endw sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 @@ -92,118 +106,65 @@ endi print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep $sleepTimer - - -$loopCnt = 0 -wait_dnode2_offline_0: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline_0 -endi - system sh/exec.sh -n dnode2 -s start -sleep $sleepTimer +$x = 0 +a0: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -$loopCnt = 0 -wait_dnode2_reready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_reready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_reready +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a0 endi sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 endi - print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 3000 -sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode3_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode3_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode3_ready +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_3 != ready then + goto step2 endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode3_ready + +sql alter database $db replica 2 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -211,44 +172,22 @@ if $data00 != $totalRows then return -1 endi - print ============== step4: stop dnode2 for checking if sync ok system sh/exec.sh -n dnode2 -s stop -sleep $sleepTimer -$loopCnt = 0 -wait_dnode2_offline: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_offline -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline -endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode2_offline +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a2 endi -sleep $sleepTimer # waitting for move master vnode of dnode2 to dnode3 # check using select sql select count(*) from $stb print data00 $data00 @@ -258,38 +197,20 @@ endi print ============== step5: restart dnode2 system sh/exec.sh -n dnode2 -s start -sleep 3000 - -$loopCnt = 0 -wait_dnode2_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_ready +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -300,40 +221,39 @@ endi print ============== step6: start dnode4 and add into cluster , then alter replica from 2 to 3, and waiting sync system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 - -sql alter database $db replica 3 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode4_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes -if $rows != 4 then - sleep 2000 - goto wait_dnode4_ready +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_4 != ready then + goto step6 endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -#$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode4Status != ready then - sleep 2000 - goto wait_dnode4_ready + +sql alter database $db replica 3 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -343,44 +263,20 @@ endi print ============== step7: alter replica from 3 to 2, and waiting sync sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_2: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 $data10_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 $data10_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 $data10_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 $data10_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$thirdDnode_2 = $data8_1 -$thirdDnode_3 = $data8_2 -$thirdDnode_4 = $data8_3 -$thirdDnode_5 = $data8_4 - -if $thirdDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a5 endi -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -390,85 +286,20 @@ endi print ============== step8: alter replica from 2 to 1, and waiting sync sql alter database $db replica 1 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_1: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$sencodDnode_2 = $data6_1 -$sencodDnode_3 = $data6_2 -$sencodDnode_4 = $data6_3 -$sencodDnode_5 = $data6_4 - -if $sencodDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a6 endi -$loopCnt = 0 -all_dnodes_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi - -sql show dnodes -if $rows != 4 then - sleep 2000 - goto all_dnodes_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode1Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode2Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode3Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode4Status != ready then - sleep 2000 - goto all_dnodes_ready -endi - -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -481,39 +312,31 @@ sql drop dnode $hostname2 sql drop dnode $hostname3 sleep $sleepTimer -$loopCnt = 0 -wait_dnode23_dropped: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode23_dropped -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $dnode2Status != null then - sleep 2000 - goto wait_dnode23_dropped -endi -if $dnode3Status != null then - sleep 2000 - goto wait_dnode23_dropped +if $rows != 3 then + goto step9 endi -if $dnode4Status != ready then - return -1 + +$x = 0 +a7: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a7 endi sleep $sleepTimer #waiting move vnode from dnode3/dnode3 to dnode4 @@ -523,3 +346,13 @@ print data00 $data00 if $data00 != $totalRows then return -1 endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index c36e9f015e16512293b3113f9e46de0fd581c4ef..73702835f4f41c74ae4161019808d26fd8a7c7a7 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 10000 @@ -107,7 +107,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_dropped endi @@ -140,3 +140,11 @@ if $data00 != $totalRows then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim index 438e4af34f4a462ea6e84e075889a4e289d35743..27b308411aea5022641ae920a3a4e8848fb1dc04 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim @@ -50,7 +50,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data @@ -60,7 +60,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $rowNum = 10 $tblNum = 16 @@ -152,7 +152,7 @@ endi print ============== step4: restart dnode4, but there are not dnode4 in cluster system sh/exec.sh -n dnode4 -s start -sleep 3000 +sleep 2000 sql show dnodes if $rows != 3 then return -1 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim index 4f80c2389ee25d06a9852d38af9df3d40d838c90..6d81effab63bb52801ab51e53ee1147326b3e851 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim index 3b208e6f1a401f855c434bd02dcab364a05694bb..d22aca07cbd3d3c6a902076103c9759209c6a966 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim index fb0650b78a6875a812fdbb1cbac4a97e48e88348..884a43bce12d1bc2c137bab17b0e780c521e327c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data to can fall disc @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 4 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim index f50081ea70abfc37223e6b3720915ecd4a5eedb9..3c74de49166067624335937d0e0486924b9fdb4f 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim index 3582a3b7b26fdeea351adbc2bb5e08aca5f9cbd6..d0399222f14cc8b8d7ac12ebca8b91549c34942d 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 @@ -106,7 +106,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -167,7 +167,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_reready endi @@ -227,7 +227,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode3_offline endi @@ -267,7 +267,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode3_reready endi @@ -343,7 +343,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode2_offline endi @@ -383,7 +383,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode2_reready endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim index 4040318801b5bf253ebf4f8619b1a715f0099c30..19b29bf342d7c8d045b16111fdc5d2ef9b2039f1 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_offline_0 endi @@ -198,7 +198,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_reready endi @@ -276,7 +276,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_offline_0 endi @@ -357,7 +357,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode23_reready_2 endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index 53209b0d5a07781ff8e22f406fc31823bdacc9b6..8e15c4f527db044d52cd725714f15b98161da2ce 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -50,7 +50,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -60,7 +60,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 @@ -110,7 +110,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -158,17 +158,19 @@ if $dnode4Vtatus != offline then sleep 2000 goto wait_dnode4_vgroup_offline endi -if $dnode3Vtatus != unsynced then +if $dnode3Vtatus != master then sleep 2000 goto wait_dnode4_vgroup_offline endi -sql_error select count(*) from $stb -sql_error insert into $tb values (now, 9988) +sql select count(*) from $stb -x s31 +s31: +#sql_error insert into $tb values (now, 9988) -x s32 +#s32: print ============== step4: restart dnode2, then create database with replica 2, and create table, insert data system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -217,7 +219,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode3_offline_0 endi diff --git a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim index 7906718b0890d6c69a1fb3e00e25ed00191c5b42..d90853d2e41e8384acad2f4dcd5a95eee8298745 100644 --- a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim @@ -39,7 +39,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 3, and create table to max tables @@ -47,10 +47,10 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start #system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 -sleep 3000 +sleep 2000 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 4 $sleepTimer = 3000 @@ -92,7 +92,7 @@ endi print ============== step3: stop dnode2 system sh/exec.sh -n dnode2 -s stop -sleep 3000 +sleep 2000 sql show mnodes print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 diff --git a/tests/script/unique/arbitrator/insert_duplicationTs.sim b/tests/script/unique/arbitrator/insert_duplicationTs.sim index a873bf02ae164e7f3bddd955adaec90568257775..4af47ca336c188c3194b9fc64925073f8fb406c2 100644 --- a/tests/script/unique/arbitrator/insert_duplicationTs.sim +++ b/tests/script/unique/arbitrator/insert_duplicationTs.sim @@ -49,7 +49,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -57,7 +57,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 $totalTableNum = 1 $sleepTimer = 3000 @@ -91,8 +91,11 @@ while $i < $tblNum $i = $i + 1 endw +sql show db.vgroups; +print d1: $data04 $data05 , d2: $data06 $data07 + sql select count(*) from $stb -print rows:$rows data00:$data00 +print rtest1==> rows:$rows data00:$data00 if $rows != 1 then return -1 endi @@ -103,6 +106,15 @@ endi $totalRows = $data00 +sql select count(*) from $stb +print test2==> rows:$rows data00:$data00 +sql select count(*) from $stb +print test3==> rows:$rows data00:$data00 +sql select count(*) from $stb +print test4==> rows:$rows data00:$data00 +sql select count(*) from $stb +print test5==> rows:$rows data00:$data00 + print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows in order to save in cache, not falling disc sql insert into $tb values ( now - 20d , -20 ) sql insert into $tb values ( now - 40d , -40 ) @@ -120,7 +132,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_offline endi @@ -153,14 +165,23 @@ if $data00 != $totalRows then return -1 endi +sql select count(*) from $stb +print data00 $data00 +sql select count(*) from $stb +print data00 $data00 +sql select count(*) from $stb +print data00 $data00 +sql select count(*) from $stb +print data00 $data00 + print ============== step5: insert two data rows: now-16d, now+16d, sql insert into $tb values ( now - 21d , -21 ) sql insert into $tb values ( now - 41d , -41 ) $totalRows = $totalRows + 2 -print ============== step5: restart dnode2, waiting sync end +print ============== step6: restart dnode2, waiting sync end system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode2_ready: $loopCnt = $loopCnt + 1 @@ -169,7 +190,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_ready endi @@ -192,9 +213,81 @@ endi sleep $sleepTimer # check using select +sleep 3000 +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 endi +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim index c11bfa62f93f6986c685cb4c3e80af7a2b28c116..0adb6b475938c1aa37f56c20c5d6327c9f89d574 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -108,7 +108,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -174,7 +174,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim index bed5cefa766cca1e35afaefabc933a3f6720df86..a0877ad89c20697e3b9a46f4512766fbd11439d8 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -204,7 +204,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim index 9fdb70142bdb511d42948efafa249a8b26ecdd49..376484a0661d89fcd38f6be790437e10c6ef2761 100644 --- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -190,7 +190,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim index 6752e0345dae1cfe8a186fcc96235600381fb7dd..9f21193400a8d7833afe8dbc5b4671c2f623778d 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim index 1280cfe7a069ffda1b639ec867eb7068b7336a9f..cb3bbb3a73f5213c346fc675049827730a6e3a01 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -176,7 +176,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim index 27c2ba4328fc99544625bf4c1e88da15bcb95ca8..8a9995f89162883c6529a3cd8e63bc764884147b 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -178,7 +178,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim index 86fb51cfa9fe6303b940cc4d199694ce94703412..6eed563bbc7c79ac35a782fbd9f2ccaa79f277d6 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -204,7 +204,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim index 4dcc5977f3a2557119abd21ca2a68b03c2423b46..2633d822c91d87ad6e5cc70370ac880fd397b889 100644 --- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -108,7 +108,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -191,7 +191,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim index 4f4cf6a2636c7559d9342b3a83c7fcf6a0a33a36..3abfc401611044b5cc7a1fb9411e4fd461beca78 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim index 98946c4b7caef7d7ee04512ac9deb1c6b7ac3983..f2acb8b90a35186395ab559ee428a182a2cba0d4 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -176,7 +176,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim index 3715be5fa9b4d9b5a623806ed3330e313edae2f1..9d0e967f4e1575195e126cd975540067038409c5 100644 --- a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim +++ b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim @@ -60,7 +60,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: replica is 1, and start 1 dnode, then create tables and insert data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $totalTableNum = 12 @@ -105,7 +105,7 @@ endi print ============== step2: add 1 new dnode, expect balanced system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 +sleep 2000 # expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2 $cnt = 0 @@ -115,7 +115,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_ready endi @@ -137,7 +137,7 @@ endi print ============== step3: stop dnode1/dnode2, modify cfg numOfMnodes to 2, and restart dnode1/dnode2 system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop -sleep 3000 +sleep 2000 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 @@ -150,7 +150,7 @@ system sh/cfg.sh -n dnode2 -c role -v 0 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ============= step4: wait dnode ready @@ -161,7 +161,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode_ready endi @@ -193,9 +193,9 @@ if $data00 != $totalRows then endi print ============== step5: stop dnode1 -sleep 5000 -system sh/exec.sh -n dnode1 -s stop sleep 3000 +system sh/exec.sh -n dnode1 -s stop +sleep 2000 $cnt = 0 wait_dnode2_master: diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim index dc8b1d015da169d0c04e6d0c23f65b8efc517562..a8c0e83cc1e5da61a473fa5b9753e90450e57bec 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim index d2c82f98ffe7aac01e42338d2dd906e19f828629..951d26635b257d1c1d3ab81889b66bb909b8038b 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim index 96aec2dcf81b84e9310ff2e29b58ba0c8282798b..e4e7f951881cf87c426c869774b2a3a548b29517 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim index a1d7d18c94549dfb0ec812077233fa2d6c96247d..0049dc6fba9bd33cd9c2e4f5b56510769dd0c60e 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index ba973cbe06d92d6bd6a007dc5bd3cdb8fa6f2bfd..499089960100f0be445552112e2b2000164c1ca0 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -140,11 +140,9 @@ if $rows != 1 then goto wait_dnode4_vgroup_offline endi print show vgroups: -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data5_2 -$dnode3Vtatus = $data7_2 +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +$dnode4Vtatus = $data05 +$dnode3Vtatus = $data07 if $dnode4Vtatus != offline then sleep 2000 @@ -198,7 +196,3 @@ if $data00 != $totalRows then sleep 2000 goto wait_table_altered endi - - - - diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim index 9b918acc8ed4fa62f6c004f2ba09d20633b386f9..10bd4fc8bd3946871605f66493e15cdde28ec257 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica3_createTable.sim b/tests/script/unique/arbitrator/sync_replica3_createTable.sim index d593577bee4bf2ec6997836905a99c327ba6f443..a0b391dd763f96fa6d340fad213943045b45ed69 100644 --- a/tests/script/unique/arbitrator/sync_replica3_createTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_createTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 20 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim index 1ef499534fe272a32556a4155a4e622a77ad1ecf..68c6ecbd6ecf7c06b5ae595c16e5b7d4b89435bd 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 20 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim index 4c7bb3d26cd8ee0012c2a8c9fadd72915e004eab..83e53eaeeb16e0988b778ef0b2515dc12c397d38 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim index 7ac424666a9ed902240451e581584897182f4265..7496541b76da3245f034924e95fd468b61135aef 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim index 3867aa3699cb84881c8b99eaa6ffa0813e8c785f..46d5e34fcbf8d70b4f678fc8e5abf311c9363ce8 100644 --- a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim +++ b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim @@ -14,6 +14,9 @@ sql alter table $stb add column f1 float $tblNum = $totalTableNum $alterTblNum = 10 +sql reset query cache +sleep 100 + $i = 1 while $i < $alterTblNum $tb = tb . $i diff --git a/tests/script/unique/big/balance.sim b/tests/script/unique/big/balance.sim index 91a7c538d27a30433aeea1fd1214e3c2d79283ec..f20939d458d512472cb822b44b5af0b378347017 100644 --- a/tests/script/unique/big/balance.sim +++ b/tests/script/unique/big/balance.sim @@ -21,7 +21,6 @@ system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 1000 print =============== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect $i = 0 @@ -92,8 +91,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show1: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -108,7 +107,7 @@ if $data2_2 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -143,7 +142,6 @@ endi print ========== step2 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 10000 print ========== step3 sql drop dnode $hostname2 @@ -151,8 +149,8 @@ sql drop dnode $hostname2 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -171,10 +169,9 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -212,8 +209,8 @@ sql drop dnode $hostname3 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -228,10 +225,9 @@ if $data2_3 != null then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -267,14 +263,31 @@ print ========== step5 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_4 != ready then + goto step5 +endi + sql alter database db replica 2 $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -289,7 +302,7 @@ if $data2_4 != 4 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -321,16 +334,14 @@ if $data00 != $totalNum then goto show5 endi - print ========== step6 -sleep 3000 sql alter database db replica 1 $x = 0 show6: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -345,7 +356,7 @@ if $data2_4 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum diff --git a/tests/script/unique/big/maxvnodes.sim b/tests/script/unique/big/maxvnodes.sim index 662d391e475f0b8afccbb37af5bae18fabe76621..10dbc8bbff1a7be19791b75d77c705ffb060e48a 100644 --- a/tests/script/unique/big/maxvnodes.sim +++ b/tests/script/unique/big/maxvnodes.sim @@ -9,14 +9,15 @@ $totalRows = $totalVnodes * $maxTables system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v $totalVnodes +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v $totalVnodes - +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 print ========== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db blocks 3 cache 1 sql use db @@ -50,7 +51,7 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show3: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 1000 then return -1 endi diff --git a/tests/script/unique/big/restartSpeed.sim b/tests/script/unique/big/restartSpeed.sim index ea4edefda8f96c94aab12d856914565f8e8115f4..bdc4a8678b2c0e766e644a69cd709ecb2cd85268 100644 --- a/tests/script/unique/big/restartSpeed.sim +++ b/tests/script/unique/big/restartSpeed.sim @@ -14,7 +14,7 @@ system sh/cfg.sh -n dnode2 -c walLevel -v 2 print ========== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db blocks 3 cache 1 sql use db diff --git a/tests/script/unique/big/tcp.sim b/tests/script/unique/big/tcp.sim index 38cb667ec03b400e29cc1017f616434b650375d4..3c5cf92c7f183a84b2f1be0ce389bf86f482a42a 100644 --- a/tests/script/unique/big/tcp.sim +++ b/tests/script/unique/big/tcp.sim @@ -15,7 +15,6 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c debugFlag -v 131 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======================== dnode1 start @@ -52,7 +51,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step3 $i sql select count(*) from $mt print ===> $data00 $data01 @@ -83,7 +82,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step5 $i sql select count(*) from $mt where tgcol < 20200 print ===> $data00 $data01 diff --git a/tests/script/unique/cluster/alter.sim b/tests/script/unique/cluster/alter.sim index 7e56707a693e96bd602c012035bebd3b1a8b7b6a..77e040c6cd9675a6e9c0ee9926f056b008bc6596 100644 --- a/tests/script/unique/cluster/alter.sim +++ b/tests/script/unique/cluster/alter.sim @@ -28,11 +28,11 @@ system sh/cfg.sh -n dnode4 -c balance -v 0 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 print ========== step2 sql create database d1 diff --git a/tests/script/unique/cluster/balance1.sim b/tests/script/unique/cluster/balance1.sim index ec0c58d41651542e03d9f2ee710a38c5744c1f82..c98687a81ceae03d20fc205a121fcfc813c74be7 100644 --- a/tests/script/unique/cluster/balance1.sim +++ b/tests/script/unique/cluster/balance1.sim @@ -84,16 +84,14 @@ endi print ============================== step2 print ========= start dnode2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 5000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -129,13 +127,12 @@ sql insert into c_b1_t3 values(1520000024031, 31) print ============================== step4 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -164,16 +161,14 @@ print dnode4 ==> $dnode4Role print ============================== step5 print ========= add dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -212,13 +207,12 @@ sql_error create dnode $hostname1 print ============================== step8 sql drop dnode $hostname3 -sleep 15000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -253,13 +247,12 @@ endi print ============================== step9 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show9 @@ -302,8 +295,8 @@ sql insert into c_b1_t4 values(1520000024041, 41) $x = 0 show10: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show10 @@ -335,7 +328,23 @@ sql_error create table c_b1_t5 (t timestamp, i int) -x error3 print ============================== step13 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 +$x = 0 +step13: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode5 $data4_5 + +if $data4_5 != ready then + goto step13 +endi sql show mnodes $dnode1Role = $data2_1 @@ -371,19 +380,35 @@ $dnode5Vnodes = $data2_5 print dnode5 $dnode2Vnodes if $dnode1Vnodes != 2 then - return -1 + goto step13 endi if $dnode4Vnodes != 2 then - return -1 + goto step13 endi if $dnode5Vnodes != 2 then - return -1 + goto step13 endi print ============================== step14 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 15000 +$x = 0 +step14: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode4 $data4_5 + +if $data4_6 != ready then + goto step14 +endi sql create database c_b1_d7 sql use c_b1_d7 @@ -406,7 +431,7 @@ sql insert into c_b1_t8 values(1520000024081, 81) $x = 0 show14: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -440,7 +465,7 @@ print ============================== step17 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql use c_b1_d1 sql select * from c_b1_d1.c_b1_t1 diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index b8477965c6a1bb8a5fca0982ef25bbf993cf6ea2..026678af7c5f7d1dfb8924152e8c26ea6dcdb2ec 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -50,12 +50,48 @@ print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 4001 sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 4001 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi + sql create database c_b2_d1 replica 2 sql use c_b2_d1 @@ -112,13 +148,12 @@ endi print ============================== step2 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show2 @@ -155,13 +190,12 @@ print ============================== step3 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 10000 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show3 @@ -209,13 +243,12 @@ endi print ============================== step4 print ========= drop dnode3 sql drop dnode $hostname3 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -266,12 +299,11 @@ print ============================== step5 print ========= start dnode3 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -303,8 +335,8 @@ print dnode5 ==> $dnode5Role print ============================== step6 system sh/exec.sh -n dnode1 -s stop -x SIGINT -print stop dnode1 and sleep 10000 -sleep 10000 +print stop dnode1 and sleep 3000 +sleep 3000 sql drop dnode $hostname1 print drop dnode1 and sleep 9000 @@ -364,7 +396,7 @@ print ============================== step12 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql select * from c_b2_d1.c_b2_t1 order by t desc print $data01 $data11 $data21 $data31 $data41 diff --git a/tests/script/unique/cluster/balance3.sim b/tests/script/unique/cluster/balance3.sim index e3b8125d8c34a7e59aaac4f3df56e4650fb6cc68..c2e9a845149cd9df78b9528115c5108c1e6531d3 100644 --- a/tests/script/unique/cluster/balance3.sim +++ b/tests/script/unique/cluster/balance3.sim @@ -7,7 +7,6 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 system sh/deploy.sh -n dnode6 -i 6 system sh/deploy.sh -n dnode7 -i 7 -system sh/deploy.sh -n dnode8 -i 8 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 @@ -16,7 +15,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode6 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode7 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode8 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0 @@ -25,7 +23,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 0 -system sh/cfg.sh -n dnode8 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode1 -c wallevel -v 1 system sh/cfg.sh -n dnode2 -c wallevel -v 1 @@ -34,19 +31,52 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 1 system sh/cfg.sh -n dnode5 -c wallevel -v 1 system sh/cfg.sh -n dnode6 -c wallevel -v 1 system sh/cfg.sh -n dnode7 -c wallevel -v 1 -system sh/cfg.sh -n dnode8 -c wallevel -v 1 print ============== step1 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 2001 - sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3001 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi sql create database c_b3_d1 replica 3 sql use c_b3_d1 @@ -89,8 +119,6 @@ $dnode2Vnodes = $data2_2 print dnode2 $dnode2Vnodes $dnode3Vnodes = $data2_3 print dnode3 $dnode3Vnodes -$dnode4Vnodes = $data2_4 -print dnode4 $dnode4Vnodes if $dnode1Vnodes != 3 then goto show1 @@ -101,30 +129,22 @@ endi if $dnode3Vnodes != 3 then goto show1 endi -if $dnode4Vnodes != null then - goto show1 -endi sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step2 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -145,21 +165,16 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step3 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show3 @@ -189,25 +204,21 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 system sh/exec.sh -n dnode2 -s stop -x SIGINT print ============================== step4 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 10000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi + sql show dnodes -x show4 $dnode1Vnodes = $data2_1 print dnode1 $dnode1Vnodes @@ -228,8 +239,10 @@ print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 print dnode4 ==> $data2_4 print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 + +if $data2_4 != slave then + goto show4 +endi print ============================== step5 print ========= drop dnode3 @@ -239,8 +252,8 @@ sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -277,16 +290,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_5 != slave then + goto show5 +endi + print ============================== step6 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 9000 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show6 @@ -299,6 +315,15 @@ print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 print dnode6 $dnode6Vnodes +if $dnode1Vnodes != 2 then + goto show6 +endi +if $dnode4Vnodes != 2 then + goto show6 +endi +if $dnode5Vnodes != 3 then + goto show6 +endi if $dnode6Vnodes != 2 then goto show6 endi @@ -315,13 +340,12 @@ print dnode7 ==> $data2_7 print ============================== step7 print ========= drop dnode4 sql drop dnode $hostname4 -sleep 9000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show7 @@ -357,16 +381,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_6 != slave then + goto show7 +endi + print ============================== step8 sql create dnode $hostname7 system sh/exec.sh -n dnode7 -s start -sleep 9000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -379,6 +406,15 @@ print dnode6 $dnode6Vnodes $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes +if $dnode1Vnodes != 2 then + goto show8 +endi +if $dnode5Vnodes != 2 then + goto show8 +endi +if $dnode6Vnodes != 3 then + goto show8 +endi if $dnode7Vnodes != 2 then goto show8 endi @@ -393,19 +429,17 @@ print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 print ============================== step9 -print ========= drop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT -print stop dnode1 and sleep 10000 -sleep 10000 -sql drop dnode $hostname1 -print drop dnode1 and sleep 9000 -sleep 9000 +$x = 0 +show9: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show mnodes -x show9 -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 @@ -414,26 +448,27 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -if $dnode1Role != offline then - return -1 +if $data2_1 != offline then + goto show9 +endi +if $data2_5 != master then + goto show9 +endi +if $data2_6 != slave then + goto show9 endi -print ============================== step9.1 -sleep 2000 -system sh/exec.sh -n dnode1 -s start - +print ============================== step10 +sql drop dnode $hostname1 $x = 0 -show9: +show10: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi +sql show mnodes -x show10 -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 @@ -442,7 +477,31 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -sql show dnodes -x show9 +if $data2_1 != null then + goto show10 +endi +if $data2_5 != master then + goto show10 +endi +if $data2_6 != slave then + goto show10 +endi +if $data2_7 != slave then + goto show10 +endi + +print ============================== step11 +system sh/exec.sh -n dnode1 -s start + +$x = 0 +show11: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes -x show11 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -451,17 +510,16 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 3 then - goto show9 + goto show11 endi if $dnode6Vnodes != 3 then - goto show9 + goto show11 endi if $dnode7Vnodes != 3 then - goto show9 + goto show11 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql show mnodes print dnode1 ==> $data2_1 @@ -472,15 +530,13 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -print ============================== step11 -print ========= add db4 - +print ============================== step12 sql create database c_b3_d4 replica 3 sql use c_b3_d4 $x = 0 create4: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -491,16 +547,14 @@ sql insert into c_b3_t4 values(1520000022043, 43) sql insert into c_b3_t4 values(1520000023042, 42) sql insert into c_b3_t4 values(1520000024041, 41) -sleep 3000 - $x = 0 -show11: +show12: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi -sql show dnodes -x show11 +sql show dnodes -x show12 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -509,21 +563,18 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 4 then - goto show11 + goto show12 endi if $dnode6Vnodes != 4 then - goto show11 + goto show12 endi if $dnode7Vnodes != 4 then - goto show11 + goto show12 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 - print ============================== step13 sql reset query cache -sleep 1000 +sleep 200 print ========= check data @@ -590,4 +641,3 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/cluster/cache.sim b/tests/script/unique/cluster/cache.sim index e23b40782870be00eb3f010cbcdba693e18b19da..7a5afae79d7fc501724ce61bc6afb388aa4df51c 100644 --- a/tests/script/unique/cluster/cache.sim +++ b/tests/script/unique/cluster/cache.sim @@ -17,7 +17,7 @@ system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/cfg.sh -n dnode2 -c monitorInterval -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database testdb @@ -33,7 +33,7 @@ while $x < 30 $x = $x + 1 endw -sleep 3000 +sleep 2000 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 diff --git a/tests/script/unique/cluster/cluster_main.sim b/tests/script/unique/cluster/cluster_main.sim index f0a9b1a2147288fb3f0d2437a507296f9517ce09..d3750be6b4cdefa602dc2987c7cab3541589bd8f 100644 --- a/tests/script/unique/cluster/cluster_main.sim +++ b/tests/script/unique/cluster/cluster_main.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $db = db1 @@ -84,7 +84,7 @@ sql select count(tbname) from $stb print select count(tbname) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_complete_create_tables endi @@ -93,12 +93,12 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 print ============== step4-1: add dnode4/dnode5 into cluster sql create dnode $hostname4 @@ -116,7 +116,7 @@ print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 @@ -152,7 +152,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes diff --git a/tests/script/unique/cluster/cluster_main0.sim b/tests/script/unique/cluster/cluster_main0.sim index 9f775c0cef183bfe3b8dd231a08f5f829fe2d3ef..48403d011b4b7603e861441728fd030de6e3c7ba 100644 --- a/tests/script/unique/cluster/cluster_main0.sim +++ b/tests/script/unique/cluster/cluster_main0.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $db = db1 @@ -85,7 +85,7 @@ sql select count(tbname) from $stb print select count(tbname) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_complete_create_tables endi @@ -94,12 +94,12 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 $loop_cnt = 0 loop_cluster_do: @@ -110,14 +110,14 @@ system sh/exec.sh -n dnode5 -s start sql create dnode $hostname4 sql create dnode $hostname5 -sleep 5000 +sleep 3000 print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 print ============== step6-1: restart dnode1 @@ -139,7 +139,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes @@ -235,7 +235,7 @@ endi print ============== step14: stop and drop dnode4/dnode5, then remove data dir of dnode4/dnode5 system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT -sleep 3000 +sleep 2000 sql drop dnode $hostname4 sql drop dnode $hostname5 system rm -rf ../../../sim/dnode4/data diff --git a/tests/script/unique/cluster/cluster_main1.sim b/tests/script/unique/cluster/cluster_main1.sim index 7796f1ea00da9d9915aba6a9cbecee0223f99fd4..a2426dc5744fe116329013e658dee48dc957a5f2 100644 --- a/tests/script/unique/cluster/cluster_main1.sim +++ b/tests/script/unique/cluster/cluster_main1.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $replica = 3 @@ -92,17 +92,17 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 print ============== step4-1: add dnode4/dnode5 into cluster sql create dnode $hostname4 sql create dnode $hostname5 -sleep 5000 +sleep 3000 $loop_cnt = 0 loop_cluster_do: @@ -116,7 +116,7 @@ print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 print ============== step6-1: restart dnode1 @@ -138,7 +138,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes diff --git a/tests/script/unique/cluster/cluster_main2.sim b/tests/script/unique/cluster/cluster_main2.sim index 48661546815a1e4c60e2b8d983f0a3e20e232529..e050ab3acf54d22becb50f6e91f83fc4eab0485b 100644 --- a/tests/script/unique/cluster/cluster_main2.sim +++ b/tests/script/unique/cluster/cluster_main2.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $replica = 3 @@ -87,7 +87,7 @@ sql select count(tbname) from $stb print select count(tbname) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_complete_create_tables endi @@ -96,17 +96,17 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 print ============== step4-1: add dnode4/dnode5 into cluster sql create dnode $hostname4 sql create dnode $hostname5 -sleep 5000 +sleep 3000 $loop_cnt = 0 @@ -120,7 +120,7 @@ print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 print ============== step6-1: restart dnode1 @@ -142,7 +142,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes diff --git a/tests/script/unique/cluster/flowctrl.sim b/tests/script/unique/cluster/flowctrl.sim index 6dc60d9fba0fd0a5f7a8088d8277c2da9e06a259..700fa0a3f195c84ed6e7903fe572c81b20b4fd21 100644 --- a/tests/script/unique/cluster/flowctrl.sim +++ b/tests/script/unique/cluster/flowctrl.sim @@ -79,14 +79,14 @@ while $x < 100 endw print =============== step3 -sleep 3000 +sleep 2000 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT print =============== step4 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start @@ -101,7 +101,7 @@ endw print =============== step6 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +sleep 2000 while $x < 300 $ms = $x . s sql insert into tb values (now + $ms , $x ) diff --git a/tests/script/unique/cluster/vgroup100.sim b/tests/script/unique/cluster/vgroup100.sim index 68a5bad6b32dc23c667900707d3e9027ac943077..656ed2ec44ebbb2d666ed2618e15d5ae34a5e525 100644 --- a/tests/script/unique/cluster/vgroup100.sim +++ b/tests/script/unique/cluster/vgroup100.sim @@ -17,7 +17,6 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 0 print ============================== step1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ============================== step2 @@ -27,8 +26,6 @@ system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 - $maxNum = 102 $maxNum = 12 @@ -92,13 +89,11 @@ print ============================== step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 print ============================== step6 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 10000 print ============================== step7 diff --git a/tests/script/unique/clusterSimCase/cluster_main.sim b/tests/script/unique/clusterSimCase/cluster_main.sim index 39607c2915da7594f7af3562ae3ccd7680b2b4aa..274ce85974bb2238bae60e6ffa9723760ae1c394 100644 --- a/tests/script/unique/clusterSimCase/cluster_main.sim +++ b/tests/script/unique/clusterSimCase/cluster_main.sim @@ -88,8 +88,8 @@ sql connect print ============== step1: add dnode2/dnode3 into cluster sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 -sleep 3000 +sleep 2000 +sleep 2000 print ============== step3: start back client-01.sim #run_back unique/clusterSimCase/client-01.sim #run_back unique/clusterSimCase/client-02.sim @@ -142,7 +142,7 @@ if $vg2Dnode2Status != master then endi -sleep 3000 +sleep 2000 print ============== step3: restart dnode3 system sh/exec.sh -n dnode3 -s start @@ -174,7 +174,7 @@ if $vg2Dnode2Status != master then goto wait_vgroup_chang_1 endi -sleep 3000 +sleep 2000 print ============== step4: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT @@ -207,7 +207,7 @@ if $vg2Dnode2Status != offline then endi -sleep 3000 +sleep 2000 print ============== step5: restart dnode2 system sh/exec.sh -n dnode2 -s start @@ -239,7 +239,7 @@ if $vg2Dnode3Status != master then goto wait_vgroup_chang_3 endi -sleep 3000 +sleep 2000 print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** **** $loop_cnt = $loop_cnt + 1 if $loop_cnt == 50 then diff --git a/tests/script/unique/db/commit.sim b/tests/script/unique/db/commit.sim index caff3c6a7898090b307d487ccdf2b8ac0dd7a1f3..661dd4505f704fff961739075247d4264d617f72 100644 --- a/tests/script/unique/db/commit.sim +++ b/tests/script/unique/db/commit.sim @@ -16,12 +16,12 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 print ========= start dnode1 as master system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ========= start other dnodes sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 print ======== step1 create db sql create database commitdb replica 1 days 7 keep 30 @@ -48,9 +48,9 @@ endi print ======== step2 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb order by ts desc print ===> rows $rows @@ -99,9 +99,9 @@ endi print ======== step5 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb print ===> rows $rows diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index fee47caadb7ab4cdab9db59b804a03eeb07a0149..c876f23de35f67b0cd2a6f0e428a825d98de7c27 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -18,13 +18,12 @@ system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 print ======== step1 sql create database db replica 3 blocks 3 @@ -38,8 +37,6 @@ while $i < 2000 $i = $i + 1 endw -sleep 2500 - sql show db.vgroups if $rows != 2 then return -1 @@ -52,7 +49,7 @@ if $rows != 0 then return -1 endi -sleep 3000 +sleep 2000 sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 @@ -73,7 +70,6 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start @@ -92,7 +88,6 @@ if $data2_1 != master then goto step3 endi -sleep 1000 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index fd92c160d9fcc8f2b5f2e93e2d656c0f48a8e80e..682444e922a5e1403ac6046b85f1c48b0f4bc78a 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -31,11 +31,29 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + $loop = 0 begin: @@ -53,7 +71,6 @@ begin: sql insert into t13 values(now, 1 ) sql create table t14 (ts timestamp, i int) sql insert into t14 values(now, 1 ) - sleep 1200 sql create table t21 (ts timestamp, i int) sql insert into t21 values(now, 1 ) @@ -63,7 +80,6 @@ begin: sql insert into t23 values(now, 1 ) sql create table t24 (ts timestamp, i int) sql insert into t24 values(now, 1 ) - sleep 1200 sql create table t31 (ts timestamp, i int) sql insert into t31 values(now, 1 ) @@ -73,7 +89,6 @@ begin: sql insert into t33 values(now, 1 ) sql create table t34 (ts timestamp, i int) sql insert into t34 values(now, 1 ) - sleep 1200 sql create table t41 (ts timestamp, i int) sql insert into t41 values(now, 1 ) @@ -83,7 +98,6 @@ begin: sql insert into t43 values(now, 1 ) sql create table t44 (ts timestamp, i int) sql insert into t44 values(now, 1 ) - sleep 1200 sql create table t51 (ts timestamp, i int) sql insert into t51 values(now, 1 ) @@ -93,7 +107,6 @@ begin: sql insert into t53 values(now, 1 ) sql create table t54 (ts timestamp, i int) sql insert into t54 values(now, 1 ) - sleep 1200 sql create table t61 (ts timestamp, i int) sql insert into t61 values(now, 1 ) @@ -103,7 +116,6 @@ begin: sql insert into t63 values(now, 1 ) sql create table t64 (ts timestamp, i int) sql insert into t64 values(now, 1 ) - sleep 1200 sql create table t71 (ts timestamp, i int) sql insert into t71 values(now, 1 ) @@ -113,7 +125,6 @@ begin: sql insert into t73 values(now, 1 ) sql create table t74 (ts timestamp, i int) sql insert into t74 values(now, 1 ) - sleep 1200 sql create table t81 (ts timestamp, i int) sql insert into t81 values(now, 1 ) @@ -123,7 +134,6 @@ begin: sql insert into t83 values(now, 1 ) sql create table t84 (ts timestamp, i int) sql insert into t84 values(now, 1 ) - sleep 1200 sql show dnodes print dnode1 openVnodes $data2_1 @@ -138,29 +148,36 @@ begin: print ======== step2 $loop system sh/exec.sh -n dnode2 -s stop - sleep 1000 print ==> drop database $db sql drop database $db print ======== step3 $loop sleep 2000 system sh/exec.sh -n dnode2 -s start - sleep 15000 - sql show dnodes + + $x = 0 + step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + + sql show dnodes -x step3 print dnode1 openVnodes $data2_1 $data4_1 print dnode2 openVnodes $data2_2 $data4_2 if $data2_1 != 0 then - return -1 + goto step3 endi if $data2_2 != 0 then - return -1 + goto step3 endi if $data4_1 != ready then - return -1 + goto step3 endi if $data4_2 != ready then - return -1 + goto step3 endi print ===> test times : $loop @@ -171,7 +188,7 @@ begin: $loop = $loop + 1 sql reset query cache - sleep 1000 + sleep 100 goto begin diff --git a/tests/script/unique/db/replica_add12.sim b/tests/script/unique/db/replica_add12.sim index e29cf1ba73b471eaab1de058cf9a26752429aebb..d46187e4456acbd39c7b1f7689388af156d04f85 100644 --- a/tests/script/unique/db/replica_add12.sim +++ b/tests/script/unique/db/replica_add12.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -27,7 +32,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -65,8 +106,6 @@ if $rows != 1 then return -1 endi -sleep 2000 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -89,10 +128,40 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 10000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ======== step3 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -137,32 +206,54 @@ if $rows != 2 then endi sql reset query cache -sleep 2000 +sleep 200 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sql select * from d1.t1 -x s51 +s51: +#sql insert into d1.t1 values(now, 3) -x s52 +s52: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +print ========= step6 +system sh/exec.sh -n dnode2 -s start + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -print ===== insert data +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -print ========= step6 -system sh/exec.sh -n dnode2 -s start -sleep 5000 +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -sql insert into d1.t1 values(now, 3) -sql insert into d2.t2 values(now, 3) -sql insert into d3.t3 values(now, 3) -sql insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(now, 3) -x step6 +sql insert into d2.t2 values(now, 3) -x step6 +sql insert into d3.t3 values(now, 3) -x step6 +sql insert into d4.t4 values(now, 3) -x step6 sql select * from d1.t1 if $rows != 3 then @@ -185,28 +276,52 @@ if $rows != 3 then endi print ========= step61 - system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +#sql insert into d1.t1 values(now, 3) -x s61 +s61: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d2.t2 -x s62 +s62: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 -sql insert into d1.t1 values(now, 5) -sql insert into d2.t2 values(now, 5) -sql insert into d3.t3 values(now, 5) -sql insert into d4.t4 values(now, 5) +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql insert into d1.t1 values(now, 5) -x step7 +sql insert into d2.t2 values(now, 5) -x step7 +sql insert into d3.t3 values(now, 5) -x step7 +sql insert into d4.t4 values(now, 5) -x step7 sql select * from d1.t1 if $rows != 4 then @@ -231,8 +346,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add13.sim b/tests/script/unique/db/replica_add13.sim index 1df49ba6582b3e1502e9b19ac3b11626175a295e..13a5c9783228761c2b431f9bb1f571c29c05f080 100644 --- a/tests/script/unique/db/replica_add13.sim +++ b/tests/script/unique/db/replica_add13.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,41 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -72,17 +111,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 -print ======== step3 $x = 0 -show3: +a1: $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 + sleep 1000 + if $x == 40 then + return -1 endi - + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +print ======== step3 sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -133,9 +195,8 @@ endi print ========= step5 sql reset query cache -sleep 1000 +sleep 100 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000013, 3) sql insert into d2.t2 values(1589529000023, 3) @@ -164,9 +225,40 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000014, 4) sql insert into d2.t2 values(1589529000024, 4) @@ -199,9 +291,40 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000015, 5) sql insert into d2.t2 values(1589529000025, 5) @@ -230,9 +353,40 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000016, 6) sql insert into d2.t2 values(1589529000026, 6) @@ -262,8 +416,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add23.sim b/tests/script/unique/db/replica_add23.sim index 5da73cd117ef218d1421d4f7f6ee7026c9e6bef3..ac0bd6d9d7b09065c832fb8ad0698b97a9f94853 100644 --- a/tests/script/unique/db/replica_add23.sim +++ b/tests/script/unique/db/replica_add23.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,42 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -72,17 +112,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 -print ======== step3 $x = 0 -show3: +a1: $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 + sleep 1000 + if $x == 40 then + return -1 endi +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +print ======== step3 sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -132,12 +195,10 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 100 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql insert into d1.t1 values(1588262400003, 3) sql insert into d2.t2 values(1588262400003, 3) sql insert into d3.t3 values(1588262400003, 3) @@ -165,9 +226,40 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400004, 4) sql insert into d2.t2 values(1588262400004, 4) @@ -196,9 +288,40 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400005, 5) sql insert into d2.t2 values(1588262400005, 5) @@ -227,9 +350,40 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400006, 6) sql insert into d2.t2 values(1588262400006, 6) @@ -259,8 +413,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_part.sim b/tests/script/unique/db/replica_part.sim index de3f6203d2f9192d8f119145c6a6b8ac6cd674f8..9880ec666c8543676c406c843d14235a09ae13ee 100644 --- a/tests/script/unique/db/replica_part.sim +++ b/tests/script/unique/db/replica_part.sim @@ -14,6 +14,10 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +25,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -60,17 +95,79 @@ if $rows != 1 then endi print ========= step2 alter db -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========= step3 system sh/exec.sh -n dnode2 -s start -sleep 10000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi print ========= step4 sql insert into d1.t1 values(now, 2) @@ -100,38 +197,99 @@ endi print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 3) -x s1 s1: -sql insert into d2.t2 values(now, 3) -x s2 -s2: -sql insert into d3.t3 values(now, 3) -x s3 -s3: -sql insert into d4.t4 values(now, 3) -x s4 -s4: +#sql insert into d2.t2 values(now, 3) -x s2 +#s2: +#sql insert into d3.t3 values(now, 3) -x s3 +#s3: +#sql insert into d4.t4 values(now, 3) -x s4 +#s4: print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 -sql insert into d1.t1 values(now, 4) -x s5 -s5: +#sql insert into d1.t1 values(now, 4) -x s5 +#s5: sql insert into d2.t2 values(now, 4) -x s6 s6: -sql insert into d3.t3 values(now, 4) -x s7 -s7: -sql insert into d4.t4 values(now, 4) -x s8 -s8: +#sql insert into d3.t3 values(now, 4) -x s7 +#s7: +#sql insert into d4.t4 values(now, 4) -x s8 +#s8: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) @@ -145,9 +303,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce21.sim b/tests/script/unique/db/replica_reduce21.sim index a64f4370c89a162a0fd6cd8235eeebeee7e03806..d3a76485f81616497a8c4859aafefda7ef371c01 100644 --- a/tests/script/unique/db/replica_reduce21.sim +++ b/tests/script/unique/db/replica_reduce21.sim @@ -14,12 +14,42 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -36,6 +66,7 @@ sql insert into d2.t2 values(now, 1) sql insert into d1.t1 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) +sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -62,7 +93,21 @@ sql create database d5 replica 1 print ========= step3 alter d1 sql alter database d1 replica 1 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + print ========= step4 query d1 sql insert into d1.t1 values(now, 2) @@ -81,9 +126,9 @@ endi return print ========= step7 drop d1 sql drop database d1 -sleep 5000 - sql reset query cache +sleep 100 + print ========= step8 sql insert into d5.t5 values(now, 2) sql insert into d2.t2 values(now, 2) @@ -112,7 +157,6 @@ endi print ======== step9 stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d5.t5 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -140,10 +184,4 @@ if $rows != 3 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce31.sim b/tests/script/unique/db/replica_reduce31.sim index 3ed0a1e3b9b98dc1911142278e31ccfad25d1eb6..5350bcc78c327ae9eb35f24e6d01901cebfb7a07 100644 --- a/tests/script/unique/db/replica_reduce31.sim +++ b/tests/script/unique/db/replica_reduce31.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -63,21 +99,99 @@ print ========= step2 alter db sql_error alter database d1 replica 1 sql_error alter database d2 replica 1 sql_error alter database d3 replica 1 +sql_error alter database d4 replica 1 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 +sql alter database d4 replica 2 + +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi -sleep 8000 +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi sql alter database d1 replica 1 sql alter database d2 replica 1 sql alter database d3 replica 1 +sql alter database d4 replica 1 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi -sleep 8000 +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show dnodes +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 + +if $data02 != 0 then + goto a1 +endi +if $data12 != 2 then + goto a1 +endi +if $data22 != 2 then + goto a1 +endi print ========= step3 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) @@ -108,7 +222,39 @@ print ========= step4 alter db sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 -sleep 8000 +sql alter database d4 replica 2 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -137,38 +283,33 @@ endi print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 -sql insert into d1.t1 values(now, 4) -x step1 -step1: -sql insert into d2.t2 values(now, 4) -x step2 -step2: -sql insert into d3.t3 values(now, 4) -x step3 -step3: -sql insert into d4.t4 values(now, 4) -x step4 -step4: +#sql insert into d1.t1 values(now, 4) -x step1 +#step1: +#sql insert into d2.t2 values(now, 4) -x step2 +#step2: +#sql insert into d3.t3 values(now, 4) -x step3 +#step3: +#sql insert into d4.t4 values(now, 4) -x step4 +#step4: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 -sql insert into d1.t1 values(now, 5) -x step5 -step5: -sql insert into d2.t2 values(now, 5) -x step6 -step6: -sql insert into d3.t3 values(now, 5) -x step7 -step7: -sql insert into d4.t4 values(now, 5) -x step8 -step8: +#sql insert into d1.t1 values(now, 5) -x step5 +#step5: +#sql insert into d2.t2 values(now, 5) -x step6 +#step6: +#sql insert into d3.t3 values(now, 5) -x step7 +#step7: +#sql insert into d4.t4 values(now, 5) -x step8 +#step8: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 sql insert into d1.t1 values(now, 6) sql insert into d2.t2 values(now, 6) @@ -183,8 +324,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce32.sim b/tests/script/unique/db/replica_reduce32.sim index 664a286be4f41284ddba42f3d053d9764e7a6408..ead265d5d54bd415d12147e5b61b04fcc5d7544f 100644 --- a/tests/script/unique/db/replica_reduce32.sim +++ b/tests/script/unique/db/replica_reduce32.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -64,7 +105,38 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ========= step3 sql insert into d1.t1 values(now, 2) @@ -93,45 +165,117 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 200 print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(now, 3) -x s41 +s41: +sql insert into d2.t2 values(now, 3) -x s42 +s42: +sql insert into d3.t3 values(now, 3) -x s43 +s43: +sql insert into d4.t4 values(now, 3) -x s44 +s44: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d1.t1 -x s45 +s45: +sql select * from d2.t2 -x s46 +s46: +sql select * from d3.t3 -x s47 +s47: +sql select * from d4.t4 -x s48 +s48: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +system sh/exec.sh -n dnode3 -s stop -x SIGINT sql reset query cache -sleep 1000 +sleep 100 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql_error insert into d1.t1 values(now, 3) -x s51 +s51: +sql_error insert into d2.t2 values(now, 3) -x s52 +s52: +sql_error insert into d3.t3 values(now, 3) -x s53 +s53: +sql_error insert into d4.t4 values(now, 3) -x s54 +s54: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) sql insert into d3.t3 values(now, 5) sql insert into d4.t4 values(now, 5) +sleep 1000 sql select * from d1.t1 print d1.t1 $rows @@ -147,9 +291,4 @@ print d4.t4 $rows system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/alternativeRole.sim b/tests/script/unique/dnode/alternativeRole.sim index af627490ba8671203d740178ae8b42e22b2c3a74..955b757f06df22c884565d0fac350456e79cb73a 100644 --- a/tests/script/unique/dnode/alternativeRole.sim +++ b/tests/script/unique/dnode/alternativeRole.sim @@ -23,14 +23,14 @@ system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 sql show dnodes print dnode1 $data5_1 diff --git a/tests/script/unique/dnode/balance1.sim b/tests/script/unique/dnode/balance1.sim index 10d185f3536c1c7b4c61bc9401655d058ae17e88..8743204a03e06d0aefc66b2f70cbdc111e317237 100644 --- a/tests/script/unique/dnode/balance1.sim +++ b/tests/script/unique/dnode/balance1.sim @@ -33,7 +33,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 sql create database d1 sql create table d1.t1 (t timestamp, i int) @@ -50,15 +49,14 @@ if $data2_1 != 1 then endi print ========== step2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -124,8 +122,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -174,8 +172,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -258,7 +256,7 @@ endi system sh/exec.sh -n dnode3 -s stop -x SIGINT sql reset query cache -sleep 1000 +sleep 100 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance2.sim b/tests/script/unique/dnode/balance2.sim index 23897df690634759de027c9b26ad0449694242a9..4c67e20c3eb5e65b0bacf0aafc3d9dc1a53d5ac3 100644 --- a/tests/script/unique/dnode/balance2.sim +++ b/tests/script/unique/dnode/balance2.sim @@ -32,7 +32,29 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database d1 replica 2 sql create table d1.t1 (t timestamp, i int) @@ -70,8 +92,8 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -90,7 +112,6 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step3 sql create dnode $hostname4 @@ -99,7 +120,7 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show3: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -156,8 +177,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -189,8 +210,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -217,10 +238,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance3.sim b/tests/script/unique/dnode/balance3.sim index 9f7f3abb8bde66fdad63351d76e6d11e64db38aa..f5558d504e34608eafb2a076a8739024e13dcff2 100644 --- a/tests/script/unique/dnode/balance3.sim +++ b/tests/script/unique/dnode/balance3.sim @@ -38,7 +38,32 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi sql create database d1 replica 3 sql create table d1.t1 (t timestamp, i int) @@ -81,7 +106,7 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -106,7 +131,6 @@ if $data2_4 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start @@ -114,8 +138,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show3: $x = $x + 1 - sleep 4000 - if $x == 15 then + sleep 1000 + if $x == 60 then return -1 endi @@ -154,8 +178,8 @@ sql insert into d3.t3 values(now+5s, 31) $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -189,8 +213,8 @@ system sh/exec.sh -n dnode6 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -216,8 +240,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -245,10 +269,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balancex.sim b/tests/script/unique/dnode/balancex.sim index da8197e22df73edfd1d5c6f02d014a45511fbae0..d2c738ee97ff8432999125abfd63d253b0546a91 100644 --- a/tests/script/unique/dnode/balancex.sim +++ b/tests/script/unique/dnode/balancex.sim @@ -23,7 +23,7 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 sql create database d1 sql create table d1.t1 (t timestamp, i int) @@ -102,7 +102,7 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show4: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -145,7 +145,7 @@ if $data2_3 != 3 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql reset query cache sleep 1000 diff --git a/tests/script/unique/dnode/data1.sim b/tests/script/unique/dnode/data1.sim index 61a991148b21b471aef906223a5f600f7db38f5f..75a484abb60c1e09a7c96f8f03902ea4f0f40aa1 100644 --- a/tests/script/unique/dnode/data1.sim +++ b/tests/script/unique/dnode/data1.sim @@ -23,7 +23,7 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 2 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ========== step2 sql create dnode $hostname2 @@ -36,7 +36,7 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 10 then return -1 endi @@ -71,7 +71,7 @@ sql insert into d1.t1 values(now+5s, 31) $x = 0 show3: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 10 then return -1 endi diff --git a/tests/script/unique/dnode/datatrans_1node.sim b/tests/script/unique/dnode/datatrans_1node.sim new file mode 100644 index 0000000000000000000000000000000000000000..a156c32672cf2936c0b69fe10242744d2dd235a3 --- /dev/null +++ b/tests/script/unique/dnode/datatrans_1node.sim @@ -0,0 +1,53 @@ + +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +print =============== step1 +sql drop database -x step1 +step1: +sql create database db +sql use db +sql create table m1 (ts timestamp, speed int) + +print =============== step 2 +$x = 0 +while $x < 10 + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into m1 values ($ms , $x ) + $x = $x + 1 +endw + +sql select * from m1 + +print $rows points data are retrieved +if $rows != 10 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT + +print =============== step 3 +system sh/move_dnode.sh dnode1 dnode2 +system sh/exec.sh -n dnode2 -s start + + +print =============== step 4 +sleep 2000 +sql connect + +sql select * from db.m1 + +print $rows points data are retrieved +if $rows != 10 then + return -1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/datatrans_3node.sim b/tests/script/unique/dnode/datatrans_3node.sim new file mode 100644 index 0000000000000000000000000000000000000000..7948eb6d3a4317ab3d829a874ced71d9f290d641 --- /dev/null +++ b/tests/script/unique/dnode/datatrans_3node.sim @@ -0,0 +1,91 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 + + +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode3 -c walLevel -v 2 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c role -v 2 +system sh/cfg.sh -n dnode2 -c role -v 2 +system sh/cfg.sh -n dnode3 -c role -v 2 + + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 + + + + +print ============== step1: start dnode1 +system sh/exec.sh -n dnode1 -s start +sleep 2000 +sql connect + +print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname2 +sql create dnode $hostname3 +sleep 2000 + +# create table +sql drop database -x step1 +step1: +sql create database db +sql use db +sql create table m1 (ts timestamp, speed int) + +# insert data +$x = 0 +while $x < 10 + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into m1 values ($ms , $x ) + $x = $x + 1 +endw + +sql select * from m1 + +print $rows points data are retrieved +if $rows != 10 then + return -1 +endi + +print ============== step3: stop cluster , then move_dnode1 ,start cluster +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +system sh/move_dnode.sh dnode1 dnode4 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +print =============== step 4 +sleep 2000 +sql connect + +sql select * from db.m1 + +print $rows points data are retrieved +if $rows != 10 then + return -1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/datatrans_3node_2.sim b/tests/script/unique/dnode/datatrans_3node_2.sim new file mode 100644 index 0000000000000000000000000000000000000000..844afc5b02f7de4e41d545ad546b9ec41943b18f --- /dev/null +++ b/tests/script/unique/dnode/datatrans_3node_2.sim @@ -0,0 +1,91 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 + + +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode3 -c walLevel -v 2 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c role -v 2 +system sh/cfg.sh -n dnode2 -c role -v 2 +system sh/cfg.sh -n dnode3 -c role -v 2 + + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 + + + + +print ============== step1: start dnode1 +system sh/exec.sh -n dnode1 -s start +sleep 2000 +sql connect + +print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname2 +sql create dnode $hostname3 +sleep 2000 + +# create table +sql drop database -x step1 +step1: +sql create database db replica 2 +sql use db +sql create table m1 (ts timestamp, speed int) + +# insert data +$x = 0 +while $x < 10 + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into m1 values ($ms , $x ) + $x = $x + 1 +endw + +sql select * from m1 + +print $rows points data are retrieved +if $rows != 10 then + return -1 +endi + +print ============== step3: stop cluster , then move_dnode1 ,start cluster +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +system sh/move_dnode.sh dnode1 dnode4 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +print =============== step 4 +sleep 2000 +sql connect + +sql select * from db.m1 + +print $rows points data are retrieved +if $rows != 10 then + return -1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/lossdata.sim b/tests/script/unique/dnode/lossdata.sim new file mode 100644 index 0000000000000000000000000000000000000000..89ba7169708eedb21c663a5e4fe0f897d42c5f43 --- /dev/null +++ b/tests/script/unique/dnode/lossdata.sim @@ -0,0 +1,165 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 2 +sql create table d1.t1 (t timestamp, i int) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 1 then + return -1 +endi +if $data2_3 != 1 then + return -1 +endi + +print ========== step3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi +if $data2_4 != 0 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step4 +sql drop dnode $hostname3 + +$i = 0 +$rowNum = 10000 + +while $i < $rowNum + $ts = 1500000000000 + $i + sql insert into d1.t1 values( $ts , $i ) + + $i = $i + 1 +endw + +print insert $rowNum finished + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 1 then + goto show4 +endi +if $data2_3 != null then + goto show4 +endi +if $data2_4 != 1 then + goto show4 +endi + +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print ========== step5 +sql select count(*) from d1.t1 +print select count(*) from d1.t1 ==> $data00 +if $data00 != $rowNum then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/m2.sim b/tests/script/unique/dnode/m2.sim new file mode 100644 index 0000000000000000000000000000000000000000..5fdf3b740081e74024f02616483af2c943d47eb9 --- /dev/null +++ b/tests/script/unique/dnode/m2.sim @@ -0,0 +1,367 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 2 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 15) +sql insert into d1.t1 values(now+2s, 14) +sql insert into d1.t1 values(now+3s, 13) +sql insert into d1.t1 values(now+4s, 12) +sql insert into d1.t1 values(now+5s, 11) + +sql create database d2 replica 2 +sql create table d2.t2 (t timestamp, i int) +sql insert into d2.t2 values(now+1s, 25) +sql insert into d2.t2 values(now+2s, 24) +sql insert into d2.t2 values(now+3s, 23) +sql insert into d2.t2 values(now+4s, 22) +sql insert into d2.t2 values(now+5s, 21) + +sql create database d3 replica 2 +sql create table d3.t3 (t timestamp, i int) +sql insert into d3.t3 values(now+1s, 35) +sql insert into d3.t3 values(now+2s, 34) +sql insert into d3.t3 values(now+3s, 33) +sql insert into d3.t3 values(now+4s, 32) +sql insert into d3.t3 values(now+5s, 31) + +sql create database d4 replica 2 +sql create table d4.t4 (t timestamp, i int) +sql insert into d4.t4 values(now+1s, 45) +sql insert into d4.t4 values(now+2s, 44) +sql insert into d4.t4 values(now+3s, 43) +sql insert into d4.t4 values(now+4s, 42) +sql insert into d4.t4 values(now+5s, 41) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 4 then + return -1 +endi +if $data2_3 != 4 then + return -1 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start +sql create dnode $hostname5 +system sh/exec.sh -n dnode5 -s start + + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 2 then + goto show3 +endi +if $data2_3 != 2 then + goto show3 +endi +if $data2_4 != 2 then + goto show3 +endi +if $data2_5 != 2 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +print ========== step4 +sql drop dnode $hostname2 + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql reset query cache +sleep 100 +system sh/exec.sh -n dnode2 -s stop -x SIGINT + +print ========== step5 +sql drop dnode $hostname3 + +$x = 0 +show5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show5 +endi +if $data2_3 != null then + goto show5 +endi +if $data2_4 != 4 then + goto show5 +endi +if $data2_5 != 4 then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql reset query cache +sleep 100 +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print ========== step6 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 11 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data41 != 15 then + return -1 +endi + +sql select * from d2.t2 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 21 then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data21 != 23 then + return -1 +endi +if $data31 != 24 then + return -1 +endi +if $data41 != 25 then + return -1 +endi + +sql select * from d3.t3 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +sql select * from d4.t4 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 41 then + return -1 +endi +if $data11 != 42 then + return -1 +endi +if $data21 != 43 then + return -1 +endi +if $data31 != 44 then + return -1 +endi +if $data41 != 45 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/m3.sim b/tests/script/unique/dnode/m3.sim new file mode 100644 index 0000000000000000000000000000000000000000..5850147d045d4e58fc8b700bcbb7d70318f0420a --- /dev/null +++ b/tests/script/unique/dnode/m3.sim @@ -0,0 +1,359 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 3 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 15) +sql insert into d1.t1 values(now+2s, 14) +sql insert into d1.t1 values(now+3s, 13) +sql insert into d1.t1 values(now+4s, 12) +sql insert into d1.t1 values(now+5s, 11) + +sql create database d2 replica 3 +sql create table d2.t2 (t timestamp, i int) +sql insert into d2.t2 values(now+1s, 25) +sql insert into d2.t2 values(now+2s, 24) +sql insert into d2.t2 values(now+3s, 23) +sql insert into d2.t2 values(now+4s, 22) +sql insert into d2.t2 values(now+5s, 21) + +sql create database d3 replica 3 +sql create table d3.t3 (t timestamp, i int) +sql insert into d3.t3 values(now+1s, 35) +sql insert into d3.t3 values(now+2s, 34) +sql insert into d3.t3 values(now+3s, 33) +sql insert into d3.t3 values(now+4s, 32) +sql insert into d3.t3 values(now+5s, 31) + +sql create database d4 replica 3 +sql create table d4.t4 (t timestamp, i int) +sql insert into d4.t4 values(now+1s, 45) +sql insert into d4.t4 values(now+2s, 44) +sql insert into d4.t4 values(now+3s, 43) +sql insert into d4.t4 values(now+4s, 42) +sql insert into d4.t4 values(now+5s, 41) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 4 then + return -1 +endi +if $data2_3 != 4 then + return -1 +endi +if $data2_4 != 4 then + return -1 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 3 then + return -1 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 2 then + return -1 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi + +print ========== step3 +sql create dnode $hostname5 +system sh/exec.sh -n dnode5 -s start + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 3 then + goto show3 +endi +if $data2_3 != 3 then + goto show3 +endi +if $data2_4 != 3 then + goto show3 +endi +if $data2_5 != 3 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 2 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +print ========== step4 +sql drop dnode $hostname2 + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show4 +endi +if $data2_3 != 4 then + goto show4 +endi +if $data2_4 != 4 then + goto show4 +endi +if $data2_5 != 4 then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 3 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql reset query cache +sleep 100 + +print ========== step5 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 11 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data41 != 15 then + return -1 +endi + +sql select * from d2.t2 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 21 then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data21 != 23 then + return -1 +endi +if $data31 != 24 then + return -1 +endi +if $data41 != 25 then + return -1 +endi + +sql select * from d3.t3 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +sql select * from d4.t4 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 41 then + return -1 +endi +if $data11 != 42 then + return -1 +endi +if $data21 != 43 then + return -1 +endi +if $data31 != 44 then + return -1 +endi +if $data41 != 45 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/monitor.sim b/tests/script/unique/dnode/monitor.sim index 1e5b0f6f56c261437ffc25c8a68139c53d7303df..b9b5e41889675ed199b67efc327477271e2f2959 100644 --- a/tests/script/unique/dnode/monitor.sim +++ b/tests/script/unique/dnode/monitor.sim @@ -23,7 +23,7 @@ system sh/cfg.sh -n dnode2 -c monitor -v 1 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 5000 +sleep 3000 sql show dnodes print dnode1 openVnodes $data3_1 @@ -65,7 +65,7 @@ sql select * from log.dn1 print $rows $rows1 = $rows -sleep 3000 +sleep 2000 sql select * from log.dn1 print $rows $rows2 = $rows @@ -79,7 +79,7 @@ sql select * from log.dn2 print $rows $rows1 = $rows -sleep 3000 +sleep 2000 sql select * from log.dn2 print $rows $rows2 = $rows diff --git a/tests/script/unique/dnode/monitor_bug.sim b/tests/script/unique/dnode/monitor_bug.sim index 3169c7cdba4c9f631a41365e25e6537dd823fc5a..efdf5e94b9648a50860dfa79715f6df8cb4c2b3a 100644 --- a/tests/script/unique/dnode/monitor_bug.sim +++ b/tests/script/unique/dnode/monitor_bug.sim @@ -15,7 +15,7 @@ system sh/cfg.sh -n dnode2 -c monitor -v 0 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 5000 +sleep 3000 sql show dnodes print dnode1 openVnodes $data2_1 @@ -64,7 +64,7 @@ sql select * from log.dn1 print $rows $rows1 = $rows -sleep 3000 +sleep 2000 sql select * from log.dn1 print $rows $rows2 = $rows diff --git a/tests/script/unique/dnode/offline1.sim b/tests/script/unique/dnode/offline1.sim index beebbfda60c03d24a774347c8ecf8c2f9a4c6c9e..9bbd14cf07891583711f8298c09da24d9fae4df4 100644 --- a/tests/script/unique/dnode/offline1.sim +++ b/tests/script/unique/dnode/offline1.sim @@ -25,7 +25,7 @@ system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql show dnodes print dnode1 $data4_1 diff --git a/tests/script/unique/dnode/offline2.sim b/tests/script/unique/dnode/offline2.sim index e8a5460de47919f42187f26a05169fa99ce47dd0..711488cf3f4dc2aebd1980c6615291b0e88dc825 100644 --- a/tests/script/unique/dnode/offline2.sim +++ b/tests/script/unique/dnode/offline2.sim @@ -32,7 +32,7 @@ system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql create database d1 replica 2 sql create table d1.t1(ts timestamp, i int) @@ -60,7 +60,7 @@ print dnode1 $data4_2 if $data4_1 != ready then return -1 endi -if $data4_2 != offline then +if $data4_2 == ready then return -1 endi @@ -84,11 +84,11 @@ system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode2 -s start sql drop dnode $hostname2 -sleep 5000 +sleep 3000 $x = 0 show4: $x = $x + 1 - sleep 5000 + sleep 3000 if $x == 10 then return -1 endi diff --git a/tests/script/unique/dnode/offline3.sim b/tests/script/unique/dnode/offline3.sim new file mode 100644 index 0000000000000000000000000000000000000000..93c75e3b13333d55aea7cb2417413a14a1c13e62 --- /dev/null +++ b/tests/script/unique/dnode/offline3.sim @@ -0,0 +1,111 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode2 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode3 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 3 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 300 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +if $data2_1 != master then + goto step1 +endi + +print ========== step2 +sql create database db replica 3 +sql use db +sql create table tb (ts timestamp, value int) +sql insert into tb values (now, 1) +sql insert into tb values (now, 2) + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step2 +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi +if $data4_3 != ready then + goto step2 +endi +if $data4_4 != null then + goto step2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/reason.sim b/tests/script/unique/dnode/reason.sim index ad61a81b9782cf3d52c65059100c879423ae4843..c685b1129d924131d38e60369d97924250b3ad66 100644 --- a/tests/script/unique/dnode/reason.sim +++ b/tests/script/unique/dnode/reason.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode2 -i 2 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -18,38 +17,75 @@ endi print ========== step2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes -print dnode1 off: $data7_1 -print dnode2 off: $data7_2 +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -sleep 3000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_2 if $data7_2 != @status msg timeout@ then - return -1 + goto step3 endi print ========== step4 sql drop dnode $hostname2 -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes if $rows != 1 then - return -1 + goto step4 endi print ========== step5 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_3 if $data7_3 != @dnodeId not match@ then - return -1 + goto step5 endi print ========== step6 @@ -58,12 +94,19 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 3 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode4 off: $data7_4 if $data7_4 != @mnEqualVn not match@ then - return -1 + goto step6 endi print ========== step7 @@ -72,12 +115,19 @@ system sh/cfg.sh -n dnode5 -c statusInterval -v 3 system sh/exec.sh -n dnode5 -s start sql create dnode $hostname5 -sleep 3000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode5 off: $data7_5 if $data7_5 != @interval not match@ then - return -1 + goto step7 endi print ========== step8 @@ -86,12 +136,19 @@ system sh/cfg.sh -n dnode6 -c balance -v 0 system sh/exec.sh -n dnode6 -s start sql create dnode $hostname6 -sleep 3000 +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode6 off: $data7_6 if $data7_6 != @balance not match@ then - return -1 + goto step8 endi print ========== step9 @@ -100,12 +157,19 @@ system sh/cfg.sh -n dnode7 -c maxTablesPerVnode -v 3000 system sh/exec.sh -n dnode7 -s start sql create dnode $hostname7 -sleep 3000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode7 off: $data7_7 if $data7_7 != @maxTabPerVn not match@ then - return -1 + goto step9 endi print ========== step10 @@ -114,12 +178,19 @@ system sh/cfg.sh -n dnode8 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode8 -s start sql create dnode $hostname8 -sleep 3000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode8 off: $data7_8 if $data7_8 != @maxVgPerDb not match@ then - return -1 + goto step10 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/dnode/remove1.sim b/tests/script/unique/dnode/remove1.sim index 6bde68b8b5c7c5c7050141645fc689f08a5fe328..6f830d2cf8d50975a494854de9932fa74f41fb5c 100644 --- a/tests/script/unique/dnode/remove1.sim +++ b/tests/script/unique/dnode/remove1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,12 +99,11 @@ endi print ========== step3 sql drop dnode $hostname2 -sleep 7001 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -101,8 +118,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -115,7 +132,6 @@ if $data2_2 != null then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step5 sql create dnode $hostname4 @@ -124,8 +140,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 903e262be79a73e76b664c077b4faa0e90139262..f97e55164b14bc62237513a057cff21c36073179 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,7 +99,6 @@ endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT sql drop dnode $hostname2 -sleep 5000 sql show dnodes print dnode1 openVnodes $data2_1 @@ -91,14 +108,20 @@ print ========== step4 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 if $data2_3 != 0 then - return -1 + goto step4 endi print ============ step 4.1 @@ -107,8 +130,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/dnode/simple.sim b/tests/script/unique/dnode/simple.sim index 014e2dd04f4e221dd036830eca95c50af03bd77e..38d8c08d75b4201fd4375afe50a83d8757ac0120 100644 --- a/tests/script/unique/dnode/simple.sim +++ b/tests/script/unique/dnode/simple.sim @@ -13,7 +13,7 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql create database d1 replica 2 sql create table d1.t1 (t timestamp, i int) @@ -44,7 +44,7 @@ endi print ========== step2 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 +sleep 2000 sql show dnodes print dnode1 openVnodes $data2_1 diff --git a/tests/script/unique/dnode/vnode_clean.sim b/tests/script/unique/dnode/vnode_clean.sim index f90f3d3fd010c7ac08305c3b38b12c7b48f5a70b..76311a6cacb2d2c06ff4125b689d3782c3e0dc98 100644 --- a/tests/script/unique/dnode/vnode_clean.sim +++ b/tests/script/unique/dnode/vnode_clean.sim @@ -45,8 +45,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -86,8 +86,8 @@ sql drop dnode $hostname2 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -106,15 +106,14 @@ endi system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========== step5 -sleep 5000 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes @@ -153,8 +152,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi @@ -184,8 +183,8 @@ sql insert into d4.t4 values(now+5s, 41) $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -208,8 +207,8 @@ sql drop dnode $hostname3 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -228,7 +227,6 @@ if $data2_4 != 4 then endi system sh/exec.sh -n dnode3 -s stop -sleep 5000 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index 8833397487aaca05342b7c41db6c979b6084ca97..1d67a7f86b2aef94a90b74d3d0e6a07dc0cf3ef2 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -8,7 +8,7 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ============================ dnode1 start @@ -69,17 +69,17 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.' -d 'show databases' 127.0.0.1:7111/admin/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: Taosd eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' 127.0.0.1:7111/admin/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":4389,"desc":"invalid taosd Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi -sleep 3000 +sleep 2000 system_content curl 127.0.0.1:7111/admin/login/root/taosdata print 9 -----> $system_content diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 526981716537ed8966e23fa3bc98662de0e06854..7d1e6b03d4547a6b0b2a6a7857000a8a6518a002 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -152,7 +152,7 @@ if $system_content != @{"status":"error","code":4515,"desc":"tag value is null"} return -1 endi -sleep 3000 +sleep 2000 print =============== step2 - insert single data system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400000,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put @@ -169,7 +169,7 @@ endi system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_web01' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then return -1 endi @@ -186,7 +186,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then return -1 endi @@ -194,7 +194,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[3]],"rows":1}@ then return -1 endi @@ -211,7 +211,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_mem_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then return -1 endi @@ -219,7 +219,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_mem_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[2]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[2]],"rows":1}@ then return -1 endi @@ -233,7 +233,7 @@ system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 134 system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[7]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[7]],"rows":1}@ then return -1 endi @@ -244,4 +244,4 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/import/replica2.sim b/tests/script/unique/import/replica2.sim index f2836fe470562dd6229a9db7f694c7cfe7826cb3..54ce28543e741a1264e1bcf76c7152bee74564c0 100644 --- a/tests/script/unique/import/replica2.sim +++ b/tests/script/unique/import/replica2.sim @@ -27,11 +27,30 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi sql create database ir2db replica 2 days 7 sql use ir2db @@ -96,9 +115,22 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -163,9 +195,22 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -193,9 +238,20 @@ endi print ================= step13 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 +endi print ================= step14 #1520000000000 @@ -214,12 +270,24 @@ endi print ================= step15 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a4 +endi + +sql select * from tb; if $rows != 52 then - return -1 + goto a4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/import/replica3.sim b/tests/script/unique/import/replica3.sim index 3a9f03a7ea5140938b21b8f5c9dc983eb701c2c2..5da9e141a5867dd2e3d61114379ede589c0fa744 100644 --- a/tests/script/unique/import/replica3.sim +++ b/tests/script/unique/import/replica3.sim @@ -27,7 +27,6 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -35,6 +34,29 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database ir3db replica 3 days 7 sql use ir3db @@ -99,9 +121,20 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -166,9 +199,21 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step10 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -196,9 +241,9 @@ endi print ================= step13 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ================= step14 #1520000000000 @@ -217,10 +262,21 @@ endi print ================= step15 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step15: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step15 +endi +sql select * from tb; if $rows != 52 then return -1 endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index 12f0013191b90cab8b85c9050f25ce182e1807b4..e5f2928748896a2aaa811ddc76bfb16b9626bf1d 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -112,7 +112,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_offline endi @@ -159,7 +159,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi @@ -194,36 +194,35 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 - -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 -if $d2v2status != master then +if $data5_4 != master then + print $data5_4 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then +print $data5_2 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then +print $data5_3 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then + print $data7_4 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then + print $data7_3 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then + print $data7_2 sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -239,7 +238,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_offline endi @@ -264,4 +263,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index 29da1fd77359146b6fb44fa8347e7b850947558d..8d063020e73be449bc95463e966d9081b0cd5be5 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -112,7 +112,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_offline endi @@ -161,7 +161,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi @@ -196,36 +196,29 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 - -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 -if $d2v2status != master then +if $data5_4 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -241,7 +234,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_offline endi @@ -266,4 +259,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim index 691b2cb568068f9e8a6f96c4cf29578a4c28854d..69e83a2c0091394b2babf76a592ab80a95ae3e6a 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim @@ -126,7 +126,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index 8a1132de2e62a6d97d37648435dd94d748a74c28..8f837b7e477ab801b296b32ddcf9a5c683c351f0 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -112,7 +112,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_offline endi @@ -159,7 +159,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi @@ -194,36 +194,29 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 - -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 -if $d2v2status != master then +if $data5_4 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -239,7 +232,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_offline endi @@ -264,4 +257,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt20.sim b/tests/script/unique/mnode/mgmt20.sim index e51d4299255248f3fb19371183536e5e62123655..8945cffab226ab5dc379057d55e562f5c3ed9cfa 100644 --- a/tests/script/unique/mnode/mgmt20.sim +++ b/tests/script/unique/mnode/mgmt20.sim @@ -11,7 +11,6 @@ system sh/cfg.sh -n dnode2 -c monitor -v 1 print ============== step1 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 3000 sql connect print ============== step2 @@ -20,8 +19,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -51,7 +50,25 @@ $d1_first = $rows sql select * from log.dn2 $d2_first = $rows -sleep 3000 +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + goto show4 +endi +if $data2_2 != slave then + goto show4 +endi + +sleep 2000 sql select * from log.dn1 $d1_second = $rows sql select * from log.dn2 diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index d55e36d7fc627c3dcfb3e9c21c9396960a0c9287..399805312ba905d55bceffe011cfe074c831684e 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -9,7 +9,6 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -26,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -47,7 +46,7 @@ print should not drop master print ============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql_error show mnodes print error of no master @@ -64,8 +63,8 @@ sql connect $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -82,13 +81,12 @@ endi print ============== step7 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 5000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/mnode/mgmt23.sim b/tests/script/unique/mnode/mgmt23.sim index d1820ef8c6a886fdbc55bda6d7b9e79a0ea51e1b..19c7b4ba762d4bf5a73c10c1afa39e927c7a1c91 100644 --- a/tests/script/unique/mnode/mgmt23.sim +++ b/tests/script/unique/mnode/mgmt23.sim @@ -25,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -65,7 +65,14 @@ endi print ============== step4 sql drop dnode $hostname2 -sleep 10000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -76,13 +83,13 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop @@ -93,7 +100,14 @@ sql create dnode $hostname2 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/exec.sh -n dnode2 -s start -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -104,19 +118,17 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 - sql_error show mnodes print ============== step7 @@ -126,7 +138,4 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt26.sim b/tests/script/unique/mnode/mgmt26.sim index 9b534cca06c89d3fa6f05ab82a88fb80514acac2..2816845052e835cf11e0ec7d4ddc71cbdee0ada1 100644 --- a/tests/script/unique/mnode/mgmt26.sim +++ b/tests/script/unique/mnode/mgmt26.sim @@ -90,7 +90,7 @@ print ============== step5 system sh/exec.sh -n dnode2 -s stop system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 sleep 6000 diff --git a/tests/script/unique/mnode/mgmt30.sim b/tests/script/unique/mnode/mgmt30.sim new file mode 100644 index 0000000000000000000000000000000000000000..d0858c0d6cdffa1cb1cd7f2ba570ae0521f412d5 --- /dev/null +++ b/tests/script/unique/mnode/mgmt30.sim @@ -0,0 +1,68 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 3000 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 3000 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 3000 + +print ============== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +print dnode3 ==> $data3_3 +if $data2_1 != master then + return -1 +endi +if $data3_2 != null then + return -1 +endi +if $data3_3 != null then + return -1 +endi + +print ============== step2 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 3000 + +sql create dnode $hostname2 +sql create dnode $hostname3 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes +$dnode1Role = $data2_1 +$dnode2Role = $data2_2 +$dnode3Role = $data2_3 +print dnode1 ==> $dnode1Role +print dnode2 ==> $dnode2Role +print dnode3 ==> $dnode3Role + +if $dnode1Role != master then + goto step2 +endi +if $dnode2Role != slave then + goto step2 +endi +if $dnode3Role != slave then + goto step2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt33.sim b/tests/script/unique/mnode/mgmt33.sim index 205e1b05d376991d72d1676262ed50c4203d3bf3..ce7cdce35d8c0463564f46d26a0711d39340c8bf 100644 --- a/tests/script/unique/mnode/mgmt33.sim +++ b/tests/script/unique/mnode/mgmt33.sim @@ -28,7 +28,14 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -39,19 +46,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -62,18 +76,25 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi print ============== step4 sql drop dnode $hostname2 -sleep 8000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -84,25 +105,30 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step5 sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -113,20 +139,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != slave then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 - -sql show mnodes +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step6 $dnode1Role = $data2_1 $dnode2Role = $data2_4 $dnode3Role = $data2_3 @@ -135,7 +167,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != offline then - return -1 + goto step6 endi #if $dnode2Role != master then # return -1 @@ -146,9 +178,15 @@ endi print ============== step7 sql drop dnode $hostname1 -sleep 8000 - -sql show mnodes +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -157,7 +195,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != null then - return -1 + goto step7 endi #if $dnode2Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmt34.sim b/tests/script/unique/mnode/mgmt34.sim index 96e3133d762362cc47f656421dfab8470e37cb2e..d8a46b0955f59273279bbbc5c89c07c05db672d7 100644 --- a/tests/script/unique/mnode/mgmt34.sim +++ b/tests/script/unique/mnode/mgmt34.sim @@ -12,7 +12,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -32,7 +31,13 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -45,22 +50,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi if $dnode4Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -73,23 +85,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi if $dnode4Role != null then - return -1 + goto step3 endi print ============== step4 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 8000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -102,21 +120,27 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi if $dnode4Role != null then - return -1 + goto step4 endi print ============== step5 sql drop dnode $hostname2 -sleep 8000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -129,28 +153,32 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi if $dnode4Role != slave then - return -1 + goto step5 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step6 sql create dnode $hostname2 -sleep 8000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -163,23 +191,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step6 endi if $dnode2Role != null then - return -1 + goto step6 endi if $dnode3Role != slave then - return -1 + goto step6 endi if $dnode4Role != slave then - return -1 + goto step6 endi print ============== step7 system sh/exec.sh -n dnode1 -s stop -sleep 4000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -190,14 +224,19 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != offline then - return -1 + goto step7 endi print ============== step8 sql drop dnode $hostname1 -sleep 8000 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step8 $dnode1Role = $data2_1 $dnode2Role = $data2_5 $dnode3Role = $data2_3 @@ -208,10 +247,10 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != null then - return -1 + goto step8 endi if $dnode2Role != slave then - return -1 + goto step8 endi #if $dnode3Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmtr2.sim b/tests/script/unique/mnode/mgmtr2.sim index 1fba912d492785900f88686097074eaafc4445af..5afb41905846bff3ce9894e928245a7d34078354 100644 --- a/tests/script/unique/mnode/mgmtr2.sim +++ b/tests/script/unique/mnode/mgmtr2.sim @@ -9,7 +9,7 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql show mnodes @@ -55,6 +55,14 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start print ============== step4 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + sql show mnodes $dnode1Role = $data2_1 @@ -65,20 +73,15 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != null then - return -1 + goto step4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/stable/dnode2_stop.sim b/tests/script/unique/stable/dnode2_stop.sim index 19c6de33b3882633acce74ca6c8f9bdbccbf7b1c..c10f04338d894106a6a543a872af69da9e49bae5 100644 --- a/tests/script/unique/stable/dnode2_stop.sim +++ b/tests/script/unique/stable/dnode2_stop.sim @@ -72,7 +72,7 @@ endi sleep 100 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 print =============== step2 sql select count(*) from $mt -x step2 @@ -84,7 +84,7 @@ sql select count(tbcol) from $mt -x step21 step21: system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print =============== step3 sql select count(tbcol) as c from $mt where ts <= 1519833840000 diff --git a/tests/script/unique/stream/metrics_balance.sim b/tests/script/unique/stream/metrics_balance.sim index 36086fe4b85b0a6cbf9710af6104d697c1bb59dc..b78cfc33a1d32c68feadc7696a449eadbdc70c36 100644 --- a/tests/script/unique/stream/metrics_balance.sim +++ b/tests/script/unique/stream/metrics_balance.sim @@ -14,8 +14,6 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 5 -system sh/cfg.sh -n dnode2 -c tableMetaKeepTimer -v 5 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 @@ -177,7 +175,7 @@ print rows0=>$r0 rows3=>$r3 rows6=>$r6 rows9=>$r9 $x = 0 show1: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -202,7 +200,7 @@ sleep 8000 $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi diff --git a/tests/script/unique/stream/metrics_vnode_stop.sim b/tests/script/unique/stream/metrics_vnode_stop.sim index 1f9cab48e5f738dabe1e07fa5b1017eb23282ac0..f1a0981d70a1f6638af6174af9bfaf84d02a0673 100644 --- a/tests/script/unique/stream/metrics_vnode_stop.sim +++ b/tests/script/unique/stream/metrics_vnode_stop.sim @@ -112,11 +112,11 @@ connectTbase2: return -1 endi sql connect -x connectTbase2 -sleep 3000 +sleep 2000 sql create dnode $hostname1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 print ======================== dnode start $dbPrefix = db diff --git a/tests/script/unique/stream/table_balance.sim b/tests/script/unique/stream/table_balance.sim index facb7df45900eff83c18e2a5ee8344d4d426cf9b..e8dec54e3ed98ebfea67d7f61d2dbbb675f9b3b4 100644 --- a/tests/script/unique/stream/table_balance.sim +++ b/tests/script/unique/stream/table_balance.sim @@ -112,7 +112,7 @@ print rows1=>$r1 rows5=>$r5 rows8=>$r8 $x = 0 show1: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -137,7 +137,7 @@ sleep 8000 $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi diff --git a/tests/script/unique/stream/table_move.sim b/tests/script/unique/stream/table_move.sim index fe19e6f402b49ee74f5a4440fb6012925a85161e..964a0c025363fd650e8051312a812fffbddaea7d 100644 --- a/tests/script/unique/stream/table_move.sim +++ b/tests/script/unique/stream/table_move.sim @@ -63,7 +63,7 @@ print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 $i = 0 $db = $dbPrefix . $i @@ -170,7 +170,7 @@ sleep 8000 $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -199,7 +199,7 @@ sleep 9000 $x = 0 show6: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi diff --git a/tests/script/unique/stream/table_vnode_stop.sim b/tests/script/unique/stream/table_vnode_stop.sim index 16350949b12e53dd452e1fba46524352a1953408..229d814e42ced35d75eec6a4ccc2fbaa66e6f654 100644 --- a/tests/script/unique/stream/table_vnode_stop.sim +++ b/tests/script/unique/stream/table_vnode_stop.sim @@ -113,11 +113,11 @@ connectTbase2: return -1 endi sql connect -x connectTbase2 -sleep 3000 +sleep 2000 sql create dnode $hostname1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 print ======================== dnode start $dbPrefix = db diff --git a/tests/script/unique/vnode/many.sim b/tests/script/unique/vnode/many.sim index 8bd6372ae9c677f5fc8475430ec3135b21409835..a9298b1cf275c24ab6ebe7fea9387a51d6d044ba 100644 --- a/tests/script/unique/vnode/many.sim +++ b/tests/script/unique/vnode/many.sim @@ -23,7 +23,37 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db1 replica 2 @@ -50,7 +80,7 @@ $lastRows4 = $rows print ======== step2 run_back unique/vnode/back_insert_many.sim -sleep 5000 +sleep 3000 print ======== step3 @@ -59,15 +89,15 @@ loop: print ======== step4 system sh/exec.sh -n dnode3 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 print ======== step5 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ======== step6 sql select count(*) from db1.tb1 diff --git a/tests/script/unique/vnode/replica2_a_large.sim b/tests/script/unique/vnode/replica2_a_large.sim index 801570dd9c18daa5611d621660e88905fa73c2b3..6f318038454a171445600e37d17852b2c2de835e 100644 --- a/tests/script/unique/vnode/replica2_a_large.sim +++ b/tests/script/unique/vnode/replica2_a_large.sim @@ -29,14 +29,14 @@ print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start -t system sh/exec.sh -n dnode3 -s start -t -sleep 3000 +sleep 2000 print ========= step1 sql create database db replica 2 @@ -56,7 +56,7 @@ $lastRows = $rows print ======== step2 #run_back unique/vnode/back_insert_many.sim run_back unique/vnode/back_insert.sim -sleep 3000 +sleep 2000 print ======== step3 diff --git a/tests/script/unique/vnode/replica2_basic2.sim b/tests/script/unique/vnode/replica2_basic2.sim index 6976353f3e292b3752b1df9f7ac1c398087a644a..c081f878dda513a3b11c9546db4dad31c3d2c5ff 100644 --- a/tests/script/unique/vnode/replica2_basic2.sim +++ b/tests/script/unique/vnode/replica2_basic2.sim @@ -22,13 +22,13 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql reset query cache @@ -126,7 +126,7 @@ endi print ========= step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 #sql insert into d1.t1 values(now, 3) #sql insert into d2.t2 values(now, 3) @@ -155,9 +155,9 @@ sleep 5000 print ========= step4 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 +sleep 3000 #sql insert into d1.t1 values(now, 4) #sql insert into d2.t2 values(now, 4) @@ -186,7 +186,7 @@ sleep 5000 print ========= step5 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) diff --git a/tests/script/unique/vnode/replica2_repeat.sim b/tests/script/unique/vnode/replica2_repeat.sim index e862d745d4fac88137d1053b1bccd22d72c5f7cc..ac68d591648b2dd66f3fdda8c70b0af40c814459 100644 --- a/tests/script/unique/vnode/replica2_repeat.sim +++ b/tests/script/unique/vnode/replica2_repeat.sim @@ -23,7 +23,38 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 2 @@ -34,7 +65,7 @@ $lastRows = $rows print ======== step2 run_back unique/vnode/back_insert.sim -sleep 3000 +sleep 2000 print ======== step3 @@ -43,15 +74,15 @@ loop: print ======== step4 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ======== step5 system sh/exec.sh -n dnode3 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 print ======== step6 sql select count(*) from db.tb @@ -64,7 +95,7 @@ print ======== step7 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_basic.sim b/tests/script/unique/vnode/replica3_basic.sim index ab5bebb50daf8a3be63d039152a020f0fff7000b..0ff42b523b8982c85bd84bb251715585a66137fc 100644 --- a/tests/script/unique/vnode/replica3_basic.sim +++ b/tests/script/unique/vnode/replica3_basic.sim @@ -16,14 +16,48 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi $N = 10 $table = table_r3 $db = db1 -sleep 3000 - print =================== step 1 sql create database $db replica 3 @@ -66,7 +100,21 @@ endi print =================== step 4 system sh/exec.sh -n dnode2 -s start -sleep 2000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step4 +endi + $y = $x + $N $expect = $N * 3 while $x < $y @@ -83,7 +131,6 @@ endi print =================== step 5 system sh/exec.sh -n dnode3 -s stop -sleep 2000 $y = $x + $N $expect = $N * 4 while $x < $y @@ -100,7 +147,21 @@ endi print =================== step 6 system sh/exec.sh -n dnode3 -s start -sleep 2000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + $y = $x + $N $expect = $N * 5 while $x < $y @@ -117,7 +178,6 @@ endi print =================== step 7 system sh/exec.sh -n dnode1 -s stop -sleep 2000 $y = $x + $N $expect = $N * 6 while $x < $y @@ -134,7 +194,21 @@ endi print =================== step 8 system sh/exec.sh -n dnode1 -s start -sleep 2000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + $y = $x + $N $expect = $N * 7 while $x < $y diff --git a/tests/script/unique/vnode/replica3_repeat.sim b/tests/script/unique/vnode/replica3_repeat.sim index 4b5c852de8a6f841ae75b9716577ae735f970b3f..636bc64f89948a8bd4ec0d3dc9dc2a1b10f50e00 100644 --- a/tests/script/unique/vnode/replica3_repeat.sim +++ b/tests/script/unique/vnode/replica3_repeat.sim @@ -26,7 +26,42 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 3 @@ -37,32 +72,32 @@ $lastRows = $rows print ======== step2 run_back unique/vnode/back_insert.sim -sleep 3000 +sleep 2000 print ======== step3 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 $x = 0 loop: print ======== step4 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s stop -sleep 5000 +sleep 3000 print ======== step5 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode4 -s stop -sleep 5000 +sleep 3000 print ======== step6 system sh/exec.sh -n dnode4 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 print ======== step7 sql select count(*) from db.tb @@ -75,7 +110,7 @@ print ======== step8 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_vgroup.sim b/tests/script/unique/vnode/replica3_vgroup.sim index 11295ba0a42cfea1ee6ff5fd50f8563ee344fc77..de4c48eca5683fc7599d545e2887ae8d2c073b82 100644 --- a/tests/script/unique/vnode/replica3_vgroup.sim +++ b/tests/script/unique/vnode/replica3_vgroup.sim @@ -16,13 +16,13 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 $N = 10 $table = table_r3 $db = db1 -sleep 3000 +sleep 2000 print =================== step 1 diff --git a/tests/script/windows/alter/metrics.sim b/tests/script/windows/alter/metrics.sim index 7d5dc779490f7591ff9330c47622e46f98fb1783..7dfda05bd0b0849fa6cbdb27e064113ad2fec595 100644 --- a/tests/script/windows/alter/metrics.sim +++ b/tests/script/windows/alter/metrics.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases @@ -368,9 +368,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d2 sql describe tb diff --git a/tests/script/windows/alter/table.sim b/tests/script/windows/alter/table.sim index 03182e613d6cea983529e75512b736acfd702765..52757da20ef226ececec64698cea7846584a73bd 100644 --- a/tests/script/windows/alter/table.sim +++ b/tests/script/windows/alter/table.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases @@ -319,9 +319,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d1 sql describe tb diff --git a/tests/script/windows/compute/avg.sim b/tests/script/windows/compute/avg.sim index b655abf1638b290bc8b6c5dd4ac8887da57c8545..7445808db5dbd293e781f66ed7010396413880bc 100644 --- a/tests/script/windows/compute/avg.sim +++ b/tests/script/windows/compute/avg.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/compute/bottom.sim b/tests/script/windows/compute/bottom.sim index dc104a8ebd4c92587f26620df0c13438b2c597bf..18f6c0bd09d1e6ee4cba734dab20f2634e587d32 100644 --- a/tests/script/windows/compute/bottom.sim +++ b/tests/script/windows/compute/bottom.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/compute/count.sim b/tests/script/windows/compute/count.sim index 9c9d8821b01c5fa86f8eefe850c0923f2681d12b..227b49a8bd61d74f242f3f505fddd599a2862cdc 100644 --- a/tests/script/windows/compute/count.sim +++ b/tests/script/windows/compute/count.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/compute/diff.sim b/tests/script/windows/compute/diff.sim index 667fcdbcff7a92c130ee5fe2253f25fd636790c3..bdc5a0e3d8a781bdccc500ef667ad6f568ad8e66 100644 --- a/tests/script/windows/compute/diff.sim +++ b/tests/script/windows/compute/diff.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/first.sim b/tests/script/windows/compute/first.sim index d6e1b1deeaccff9fa5199eed86b1d85d7c541c29..a83939c2a629e13eb474b6b936e7e4b409d7329e 100644 --- a/tests/script/windows/compute/first.sim +++ b/tests/script/windows/compute/first.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/interval.sim b/tests/script/windows/compute/interval.sim index 4bf548ccf21ad1cf89d6688d1921b9742becbe1f..feacce16066a4b5a06268033dedefe327a9f6fa8 100644 --- a/tests/script/windows/compute/interval.sim +++ b/tests/script/windows/compute/interval.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/last.sim b/tests/script/windows/compute/last.sim index 63d4d3ecbdd9cd000e7a10a814aa19bb741c6880..379a5ae64a48b467ffdae816066e8bc3c956d50c 100644 --- a/tests/script/windows/compute/last.sim +++ b/tests/script/windows/compute/last.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/leastsquare.sim b/tests/script/windows/compute/leastsquare.sim index 69c8fb377b00ad28013ac81f07818151bf387791..20353a409e166243212c072d5596492e251e984b 100644 --- a/tests/script/windows/compute/leastsquare.sim +++ b/tests/script/windows/compute/leastsquare.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/max.sim b/tests/script/windows/compute/max.sim index e480736550781babf036a3ab4af40f81753dc8b6..641b31fe362712e09081f105bacdb11f41f2bb99 100644 --- a/tests/script/windows/compute/max.sim +++ b/tests/script/windows/compute/max.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/min.sim b/tests/script/windows/compute/min.sim index 1ff637cecd27c979bb7c4a978d806726c8c62ff6..3528f573ce4c65ac03ac0c0d6c5502b33979cb17 100644 --- a/tests/script/windows/compute/min.sim +++ b/tests/script/windows/compute/min.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/percentile.sim b/tests/script/windows/compute/percentile.sim index 5e327055a8bfd563691c04ff5cb9ea7fbd9dddc5..fa6212f013b4ec0aa92c06842263fd32be05e1e9 100644 --- a/tests/script/windows/compute/percentile.sim +++ b/tests/script/windows/compute/percentile.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/stddev.sim b/tests/script/windows/compute/stddev.sim index 2aa481248ae511ae0aaa3e2621c05eb9d4ea71e2..eea6c8aa0529d0f8dba9907de76017751fbf3d6f 100644 --- a/tests/script/windows/compute/stddev.sim +++ b/tests/script/windows/compute/stddev.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/sum.sim b/tests/script/windows/compute/sum.sim index 30e98a5b2596306865d774267936e4380a16d2d1..a429ce99e0b2faea0d594cc97c7f232b11c5e781 100644 --- a/tests/script/windows/compute/sum.sim +++ b/tests/script/windows/compute/sum.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/top.sim b/tests/script/windows/compute/top.sim index 9590997ef76b396f37102b46140b8ff58598fb9e..65e448b0fa5fb1dffcaf1e63129099c57a4a9a02 100644 --- a/tests/script/windows/compute/top.sim +++ b/tests/script/windows/compute/top.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/db/basic.sim b/tests/script/windows/db/basic.sim index fffde94d66f1ae30b7632c66ff6444cfe3ba1934..914e456fe1e3a52ca107275b30995f88c37b0415 100644 --- a/tests/script/windows/db/basic.sim +++ b/tests/script/windows/db/basic.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/db/len.sim b/tests/script/windows/db/len.sim index 5afa2496dde23b624b6d54afd18f096c59d47f46..3356165117d3986d9eef4e7dced3bb1b61dd36da 100644 --- a/tests/script/windows/db/len.sim +++ b/tests/script/windows/db/len.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/field/2.sim b/tests/script/windows/field/2.sim index 8ac6fa1a1b2dd85715b536efdb14dc8b6e6d79a9..e258fbe80efbdde2d1b8f289f5302b74adb8b0f7 100644 --- a/tests/script/windows/field/2.sim +++ b/tests/script/windows/field/2.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/3.sim b/tests/script/windows/field/3.sim index 331e930b3186ad9541912f87e5342046046850cf..e3fe0b0b1162548ccd8ce0d4e5d5606a628abcef 100644 --- a/tests/script/windows/field/3.sim +++ b/tests/script/windows/field/3.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/4.sim b/tests/script/windows/field/4.sim index c6224c46ee2c0064c5fa5ea14a67310fee11d423..aabfe2be2ce76d5fa52fcde85c129b0f4bdf75ac 100644 --- a/tests/script/windows/field/4.sim +++ b/tests/script/windows/field/4.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/5.sim b/tests/script/windows/field/5.sim index d1f40059d027fba7797967c7b65d86457f4fd3b9..874730ff1c9cd341e8c3cc1f9c8d33b228d5b3ad 100644 --- a/tests/script/windows/field/5.sim +++ b/tests/script/windows/field/5.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/6.sim b/tests/script/windows/field/6.sim index 98071f87df11ccd6d5984ce9065485bf2931a6ee..77277df35b2a41bbf7ab759763b9601054b0d5d6 100644 --- a/tests/script/windows/field/6.sim +++ b/tests/script/windows/field/6.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/bigint.sim b/tests/script/windows/field/bigint.sim index bef571f4456dc6a05d6844b28128df4137c39bf4..f912ee968b0955e339340e9ee712b99eb954914a 100644 --- a/tests/script/windows/field/bigint.sim +++ b/tests/script/windows/field/bigint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/binary.sim b/tests/script/windows/field/binary.sim index 72a356e684bbada0ec1b81ca58303a3f885eca4f..aa641878e4c49463aca22ab3f57b43a558e955f7 100644 --- a/tests/script/windows/field/binary.sim +++ b/tests/script/windows/field/binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/bool.sim b/tests/script/windows/field/bool.sim index abc970264dae1346bb2177516b4056c024b69da2..3ef56a1d95082faf54f1e34db4009b8af0e80c6c 100644 --- a/tests/script/windows/field/bool.sim +++ b/tests/script/windows/field/bool.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/double.sim b/tests/script/windows/field/double.sim index e805e0373b83ab440a846139681d040a6c4b7ad1..ef404d28dc306edaf2fe6be00eb83dfb484c23a3 100644 --- a/tests/script/windows/field/double.sim +++ b/tests/script/windows/field/double.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/float.sim b/tests/script/windows/field/float.sim index 4178ab4e1e260ba9251466c53c0cd4da10ee0dc3..8435f31c1fbc3353f87f8abc620184919b32ccbc 100644 --- a/tests/script/windows/field/float.sim +++ b/tests/script/windows/field/float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/int.sim b/tests/script/windows/field/int.sim index 05dc19094d85110672201436f99f19c19b425db8..781b9394849192725ba5d46e3d669d53838501b7 100644 --- a/tests/script/windows/field/int.sim +++ b/tests/script/windows/field/int.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/single.sim b/tests/script/windows/field/single.sim index 6422b7f6971870d72b6272c77d6109484501eb5e..b6b44020915e57b130c77f2c40e1a7a2e4e1342c 100644 --- a/tests/script/windows/field/single.sim +++ b/tests/script/windows/field/single.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/smallint.sim b/tests/script/windows/field/smallint.sim index 8bf41f45a5e62e584b39daa1ecde34691a922eb2..5f1839226b1d3aabfebfe9b30ad3f8b02d1e21a9 100644 --- a/tests/script/windows/field/smallint.sim +++ b/tests/script/windows/field/smallint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/tinyint.sim b/tests/script/windows/field/tinyint.sim index 16c19ba38d58c407fd91ff43654b502fe9bb2d36..c90ff3f932a3eeb52e1ff607ac0bcacf21fecfe4 100644 --- a/tests/script/windows/field/tinyint.sim +++ b/tests/script/windows/field/tinyint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/import/basic.sim b/tests/script/windows/import/basic.sim index 491b4f8b3445e616d75da1ca70058b1fe49089f9..ee19893ae21fd8c97eeb9aa5898ead86b8b8d19f 100644 --- a/tests/script/windows/import/basic.sim +++ b/tests/script/windows/import/basic.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/basic.sim b/tests/script/windows/insert/basic.sim index 54cbd3f4d9a8ee068ff4595f15f0642ca3e92674..a516f80e1013a40a8889ca939ca650360e4fc976 100644 --- a/tests/script/windows/insert/basic.sim +++ b/tests/script/windows/insert/basic.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block1_file.sim b/tests/script/windows/insert/query_block1_file.sim index 388ed061e5c583ec46113811542b2f87267b860e..62b74ac19c18a624d41941e8c51a3e8fb87ddee0 100644 --- a/tests/script/windows/insert/query_block1_file.sim +++ b/tests/script/windows/insert/query_block1_file.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block1_memory.sim b/tests/script/windows/insert/query_block1_memory.sim index 9e4fc68d093d07d5c43797efc28265de750c376c..6c3e58d70e992a79bfcd84a670616b9b3441393f 100644 --- a/tests/script/windows/insert/query_block1_memory.sim +++ b/tests/script/windows/insert/query_block1_memory.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block2_file.sim b/tests/script/windows/insert/query_block2_file.sim index 9fd443447612917fd0ba85a9d8a591a7a384b87c..164c8a1124eb7648c2cb0d39a1dba3839849aab4 100644 --- a/tests/script/windows/insert/query_block2_file.sim +++ b/tests/script/windows/insert/query_block2_file.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block2_memory.sim b/tests/script/windows/insert/query_block2_memory.sim index ede7f3efc6e807492845786f5979e14a6ba337c6..f3ceb503acfebae398856d1a2d085fd0e3a0ce2d 100644 --- a/tests/script/windows/insert/query_block2_memory.sim +++ b/tests/script/windows/insert/query_block2_memory.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_file_memory.sim b/tests/script/windows/insert/query_file_memory.sim index 083beb4ac5f3983eb90a0d38811398dff510c0a7..d9752c40c9cc7c264a86e245eb3877fc47783b2a 100644 --- a/tests/script/windows/insert/query_file_memory.sim +++ b/tests/script/windows/insert/query_file_memory.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_multi_file.sim b/tests/script/windows/insert/query_multi_file.sim index 465970f9426440262f2a694c510ba40d184f7f64..ba617ce63c6886bc978fe732222a54a7a99ad4d5 100644 --- a/tests/script/windows/insert/query_multi_file.sim +++ b/tests/script/windows/insert/query_multi_file.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/binary.sim b/tests/script/windows/table/binary.sim index 64a081c72f5e4e5bd26013df03d7667211b4dce5..5efa0bc6667c4f2418e53f341315bf6529527455 100644 --- a/tests/script/windows/table/binary.sim +++ b/tests/script/windows/table/binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/bool.sim b/tests/script/windows/table/bool.sim index 9486c422215f39fac909af7620452dd3443eef8d..0d185d31e81c34d7289d753cce63ec419612646b 100644 --- a/tests/script/windows/table/bool.sim +++ b/tests/script/windows/table/bool.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/column_name.sim b/tests/script/windows/table/column_name.sim index fffb1334e5e317e6d22155abf4624463bfce9989..6f9f954461c6faaddb01ab8e327c008a86809bbd 100644 --- a/tests/script/windows/table/column_name.sim +++ b/tests/script/windows/table/column_name.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/column_num.sim b/tests/script/windows/table/column_num.sim index d182696ce08176f7d19c74858c069f1cdf73dd4c..395ee02cdea0b90fe8cd248af63cc950c090c283 100644 --- a/tests/script/windows/table/column_num.sim +++ b/tests/script/windows/table/column_num.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/column_value.sim b/tests/script/windows/table/column_value.sim index c59e7af8baae9c0b9b5140a624bd9b4c0c97c385..8db85f16ad876deff5c1a4b445351c83e9c5a3b7 100644 --- a/tests/script/windows/table/column_value.sim +++ b/tests/script/windows/table/column_value.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/db.table.sim b/tests/script/windows/table/db.table.sim index 97a9e6fbe94db8fec01db3db5e0a0c0335788842..0e207c998276fe4e33d0f875155d971506aadb70 100644 --- a/tests/script/windows/table/db.table.sim +++ b/tests/script/windows/table/db.table.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/double.sim b/tests/script/windows/table/double.sim index 93bf3bb1490d21d18c56bea0fc2e2a6535d59b43..b88ac4a199c9394422c7427eda6d824ec524e667 100644 --- a/tests/script/windows/table/double.sim +++ b/tests/script/windows/table/double.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/float.sim b/tests/script/windows/table/float.sim index 684f78a386f886b3f2c6f1c2e8510ebe2a1a043f..741525830d9a5ea38d82420e9affcb8015dab1db 100644 --- a/tests/script/windows/table/float.sim +++ b/tests/script/windows/table/float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/table.sim b/tests/script/windows/table/table.sim index 985620152afae21b4c7c239bb80196bbe8facca2..13d157627720e9f5c8a355255782b10ea43b4a58 100644 --- a/tests/script/windows/table/table.sim +++ b/tests/script/windows/table/table.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ============================ dnode1 start sql show databases diff --git a/tests/script/windows/table/table_len.sim b/tests/script/windows/table/table_len.sim index 367f1c6895c80fb019a68bc36f0921becf1f30c7..72ed549466a61e984a0d9f287639a590c7d07f79 100644 --- a/tests/script/windows/table/table_len.sim +++ b/tests/script/windows/table/table_len.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/tag/3.sim b/tests/script/windows/tag/3.sim index 63a8766727fbdc14ce4504949a48bad7be6b5f64..5479be158ba4ab0be883d1f0eb26d2eb09aaafa1 100644 --- a/tests/script/windows/tag/3.sim +++ b/tests/script/windows/tag/3.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/4.sim b/tests/script/windows/tag/4.sim index 7e9af7ece7ebc8f3548dcacd3c26b094a9990160..17552010b0382db3e9cc7e0bafdecdefff03dd96 100644 --- a/tests/script/windows/tag/4.sim +++ b/tests/script/windows/tag/4.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/5.sim b/tests/script/windows/tag/5.sim index 5dc128a0e0eb68eb2a4c227ade718708224c5f8c..f06d78e7b5b4082699349e12ee99c96e110fa91b 100644 --- a/tests/script/windows/tag/5.sim +++ b/tests/script/windows/tag/5.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/6.sim b/tests/script/windows/tag/6.sim index 12e9c597f0deb9ec00ef70ae9cccb761f79cfbc6..64cb9df6f030d4c0314d533fb6239daf7ba3586e 100644 --- a/tests/script/windows/tag/6.sim +++ b/tests/script/windows/tag/6.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/add.sim b/tests/script/windows/tag/add.sim index 0a1416b68c34245a1ef6553a7d19b7601b34de79..02d027ccf4e43c472a1d8b60daa99d254b2df35c 100644 --- a/tests/script/windows/tag/add.sim +++ b/tests/script/windows/tag/add.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bigint.sim b/tests/script/windows/tag/bigint.sim index d988ad1fdc6dca3616785a7646efc6fbf9f0d466..ebb67d452c82586d31f673d7c9c5c78249285795 100644 --- a/tests/script/windows/tag/bigint.sim +++ b/tests/script/windows/tag/bigint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/binary.sim b/tests/script/windows/tag/binary.sim index 9dc18cfa94bba50e748edc48beabc40f9445243f..c59039b6a6d8ba5d05a653c1f5d40de170e89f15 100644 --- a/tests/script/windows/tag/binary.sim +++ b/tests/script/windows/tag/binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/binary_binary.sim b/tests/script/windows/tag/binary_binary.sim index ba688aa80e234e4ffcf117154d0ff45827ddb02f..361a6edb8b3f6d9440a1796bed5ca7c431bcf115 100644 --- a/tests/script/windows/tag/binary_binary.sim +++ b/tests/script/windows/tag/binary_binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bool.sim b/tests/script/windows/tag/bool.sim index a7e5d909c598bc748bd431ba2a1c5d0e4db15ec6..adf12338d089c41fea9e23b65b8dee03f8eb5b46 100644 --- a/tests/script/windows/tag/bool.sim +++ b/tests/script/windows/tag/bool.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bool_binary.sim b/tests/script/windows/tag/bool_binary.sim index 639f6c5f2f6f80ffe9c9237c0352c4951a1155a3..064677ee407ee9851e45ea3c743b5d95c66e8545 100644 --- a/tests/script/windows/tag/bool_binary.sim +++ b/tests/script/windows/tag/bool_binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bool_int.sim b/tests/script/windows/tag/bool_int.sim index 900cc9e8a181584cbabb5cee2cd7e048449f7f4f..ef5cd27553161345e95d1ca1b0f414ed16394cb1 100644 --- a/tests/script/windows/tag/bool_int.sim +++ b/tests/script/windows/tag/bool_int.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/change.sim b/tests/script/windows/tag/change.sim index 75a976bbb1553960bfd77f6470e8cddabf9db110..4126ea1181e8b554cac2833eb66c27f3d4290071 100644 --- a/tests/script/windows/tag/change.sim +++ b/tests/script/windows/tag/change.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases @@ -175,7 +175,7 @@ sql alter table $mt change tag tgcol3 tgcol9 sql alter table $mt change tag tgcol5 tgcol10 sql alter table $mt change tag tgcol6 tgcol11 -sleep 5000 +sleep 3000 sql reset query cache print =============== step2 diff --git a/tests/script/windows/tag/column.sim b/tests/script/windows/tag/column.sim index 9f5bfce07e724d6aaec5268f6d85a67805f92209..40159bcae347e1292e2c327110bc4db6b7176bbb 100644 --- a/tests/script/windows/tag/column.sim +++ b/tests/script/windows/tag/column.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/create.sim b/tests/script/windows/tag/create.sim index 6a76c93d83db0afa3e9e9b2de00d614fad66d9e6..62dc8a7a21c442b0a38fd067da0d55ad5c8ed8ef 100644 --- a/tests/script/windows/tag/create.sim +++ b/tests/script/windows/tag/create.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/delete.sim b/tests/script/windows/tag/delete.sim index 9e8ea9aba03c99d116d0923bd27006c34c0253a2..2b503fdf47ac2eaa5ea18c8c6af1bbbe05e9c44c 100644 --- a/tests/script/windows/tag/delete.sim +++ b/tests/script/windows/tag/delete.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases @@ -409,7 +409,7 @@ sql alter table $mt drop tag tgcol3 sql alter table $mt drop tag tgcol4 sql alter table $mt drop tag tgcol6 -sleep 5000 +sleep 3000 print =============== step2 $i = 2 diff --git a/tests/script/windows/tag/double.sim b/tests/script/windows/tag/double.sim index 5445b1dbea083ea6664350c66b6342e606213678..4381aa20f939d79ac81804a5be54952db7a27ba4 100644 --- a/tests/script/windows/tag/double.sim +++ b/tests/script/windows/tag/double.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/filter.sim b/tests/script/windows/tag/filter.sim index f704f32cd2b5ead591ef76ed2237099273ba95a4..802e9a312f8cccbc34bb537dbba5ec3ab13b11d5 100644 --- a/tests/script/windows/tag/filter.sim +++ b/tests/script/windows/tag/filter.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/float.sim b/tests/script/windows/tag/float.sim index 64424c1e209bea2fc3a8c3aaa20dec4e03539587..8df44c24a56d8c013af7c63aa9d45b720069cbaa 100644 --- a/tests/script/windows/tag/float.sim +++ b/tests/script/windows/tag/float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/int.sim b/tests/script/windows/tag/int.sim index 7d7b5271d194df316b1cc1ac6cd4db9e45bbe2ff..dbff8c15b6aad716a1c90ad5c74269f955c7307d 100644 --- a/tests/script/windows/tag/int.sim +++ b/tests/script/windows/tag/int.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/int_binary.sim b/tests/script/windows/tag/int_binary.sim index 1dd4771605046b8b54d89d54155f41fa9f751b69..94aa9eb7f40441f5cc4e399ac8a4f5f5ce396beb 100644 --- a/tests/script/windows/tag/int_binary.sim +++ b/tests/script/windows/tag/int_binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/int_float.sim b/tests/script/windows/tag/int_float.sim index cdb9032d8cc39780f7b71297901914e86bfc1ad2..9789c9ea06640432e8866a787ec0389ed3d239c2 100644 --- a/tests/script/windows/tag/int_float.sim +++ b/tests/script/windows/tag/int_float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/set.sim b/tests/script/windows/tag/set.sim index 16103c6ce86cf62c8e8a228bebeaf931190435c4..54b87c7d0c1c4eac55a16d42b61979329a31167d 100644 --- a/tests/script/windows/tag/set.sim +++ b/tests/script/windows/tag/set.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/smallint.sim b/tests/script/windows/tag/smallint.sim index dbab4f2d43c86578c7726ca8db23863ec24f335a..bc668b164d53309512d69434e021f5744da2a253 100644 --- a/tests/script/windows/tag/smallint.sim +++ b/tests/script/windows/tag/smallint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/tinyint.sim b/tests/script/windows/tag/tinyint.sim index 7a0237c0d9dd9a02845774ad5102226955c83b38..44fc9ba4dc0d0f4a670c3c03abf36a38f658f878 100644 --- a/tests/script/windows/tag/tinyint.sim +++ b/tests/script/windows/tag/tinyint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/vector/metrics_field.sim b/tests/script/windows/vector/metrics_field.sim index e7c926e141bc90531adea7dac389a7c9f59d7fdf..dfaa7e1d9978357f5858e529ebac40941dd6bf2c 100644 --- a/tests/script/windows/vector/metrics_field.sim +++ b/tests/script/windows/vector/metrics_field.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_mix.sim b/tests/script/windows/vector/metrics_mix.sim index 3d94a9638520961119740ba14628dca4143d6040..111fdebb05735057a922ce313d9c595a7d8491ff 100644 --- a/tests/script/windows/vector/metrics_mix.sim +++ b/tests/script/windows/vector/metrics_mix.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_query.sim b/tests/script/windows/vector/metrics_query.sim index c292c6b306da676dfcc7e94fc8db30b1533848db..45e734f468deb4fb1567260fdde8b072bdb2a3b2 100644 --- a/tests/script/windows/vector/metrics_query.sim +++ b/tests/script/windows/vector/metrics_query.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_tag.sim b/tests/script/windows/vector/metrics_tag.sim index f51a449d7171fcc1ad277b16d37be4e03e23f33f..80c204fa10e963fa264aa8f64d116cfc5ee1277f 100644 --- a/tests/script/windows/vector/metrics_tag.sim +++ b/tests/script/windows/vector/metrics_tag.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_time.sim b/tests/script/windows/vector/metrics_time.sim index 716f49d1e5ffeff106914626ac293e8cf0cc0c9e..c127fe78fc1acf712cc5505a680477c0e52d8514 100644 --- a/tests/script/windows/vector/metrics_time.sim +++ b/tests/script/windows/vector/metrics_time.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/multi.sim b/tests/script/windows/vector/multi.sim index 415384d2434ac0f54d3db639421406bc5c854859..ff63cda9a5777a3f20102aafb388f9a7b491a972 100644 --- a/tests/script/windows/vector/multi.sim +++ b/tests/script/windows/vector/multi.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/single.sim b/tests/script/windows/vector/single.sim index f3f3862e54291fcd9cbeddcc684f94c055d78b19..fb3a52760b4efb5133998a9d71d0b7ce2957e69b 100644 --- a/tests/script/windows/vector/single.sim +++ b/tests/script/windows/vector/single.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_field.sim b/tests/script/windows/vector/table_field.sim index 0c5df695fb25a6e0f13a5f3e309ad5b203b94962..10c5148243793adbd01221743cbda673ae7ce651 100644 --- a/tests/script/windows/vector/table_field.sim +++ b/tests/script/windows/vector/table_field.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_mix.sim b/tests/script/windows/vector/table_mix.sim index 3d660b5611e770ce99a934e11847dd23584387b5..7418cb453df7fc627acc9f34598db6ce7db144d9 100644 --- a/tests/script/windows/vector/table_mix.sim +++ b/tests/script/windows/vector/table_mix.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_query.sim b/tests/script/windows/vector/table_query.sim index 27fcd0f65450380411a7f18c5db2099c6d8c93a3..7654688b269eb16c278c000d9da565515512062a 100644 --- a/tests/script/windows/vector/table_query.sim +++ b/tests/script/windows/vector/table_query.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_time.sim b/tests/script/windows/vector/table_time.sim index 8a3804c6196375c84a2a49e19f3b42c8d7b57ec8..bea9d41d1bcdfa77acd5eaf52a2de22353b075d9 100644 --- a/tests/script/windows/vector/table_time.sim +++ b/tests/script/windows/vector/table_time.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 6cdd63b42d28437dded9dc7f99a3888b232c16fc..9ed58a90d1ba6bd781d306070e03f56b75984093 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -1,29 +1,30 @@ @echo off echo TDengine in windows -echo Start TDengine Testing Case ... +rem echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" -echo SCRIPT_DIR: %SCRIPT_DIR% +rem echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" -set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" -echo BUILD_DIR: %BUILD_DIR% +set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" +set "TSIM=%BUILD_DIR%tsim" +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% -set "SIM_DIR=%~dp0..\..\sim" -echo SIM_DIR: %SIM_DIR% +set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" +rem echo SIM_DIR: %SIM_DIR% -set "TSIM_DIR=%~dp0..\..\sim\tsim" -echo TSIM_DIR: %TSIM_DIR% +set "TSIM_DIR=%SIM_DIR%tsim\" +rem echo TSIM_DIR: %TSIM_DIR% -set "CFG_DIR=%~dp0..\..\sim\tsim\cfg" -echo CFG_DIR: %CFG_DIR% +set "CFG_DIR=%TSIM_DIR%cfg\" +rem echo CFG_DIR: %CFG_DIR% -set "LOG_DIR=%~dp0..\..\sim\tsim\log" -echo LOG_DIR: %LOG_DIR% +set "LOG_DIR=%TSIM_DIR%log\" +rem echo LOG_DIR: %LOG_DIR% -set "TAOS_CFG=%~dp0..\..\sim\tsim\cfg\taos.cfg" -echo TAOS_CFG: %TAOS_CFG% +set "TAOS_CFG=%CFG_DIR%taos.cfg" +rem echo TAOS_CFG: %TAOS_CFG% if not exist %SIM_DIR% mkdir %SIM_DIR% if not exist %TSIM_DIR% mkdir %TSIM_DIR% @@ -32,29 +33,30 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -echo firstEp %FIRSTEP% > %TAOS_CFG% -echo serverPort 6030 >> %TAOS_CFG% -echo wal 2 >> %TAOS_CFG% -echo asyncLog 0 >> %TAOS_CFG% -echo locale en_US.UTF-8 >> %TAOS_CFG% -echo logDir %LOG_DIR% >> %TAOS_CFG% -echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% -echo numOfLogLines 100000000 >> %TAOS_CFG% -echo tmrDebugFlag 131 >> %TAOS_CFG% -echo rpcDebugFlag 143 >> %TAOS_CFG% -echo cDebugFlag 143 >> %TAOS_CFG% -echo qdebugFlag 143 >> %TAOS_CFG% -echo udebugFlag 143 >> %TAOS_CFG% - -set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=192.168.1.182" +rem set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" + +echo firstEp %fqdn% > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% +echo serverPort 7100 >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% +echo numOfLogLines 100000000 >> %TAOS_CFG% +echo rpcDebugFlag 135 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo cDebugFlag 135 >> %TAOS_CFG% +echo udebugFlag 135 >> %TAOS_CFG% +echo wal 0 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo enableCoreFile 1 >> %TAOS_CFG% + +set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" -if "%1" == "-h" set "FIRSTEP=%2" -if "%3" == "-f" set "FILE_NAME=%4" -if "%3" == "-h" set "FIRSTEP=%4" -echo FILE_NAME: %FILE_NAME% -echo FIRSTEP: %FIRSTEP% +rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% -%tsim% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file +%TSIM% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file diff --git a/tests/test-all.sh b/tests/test-all.sh index 14b649eddfe92e8c557f81d27a0d1b95b35602c3..4f7afe7d17bc57d6f62a5d68cac3277914dedb50 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -7,21 +7,92 @@ GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' +tests_dir=`pwd` +IN_TDINTERNAL="community" + +function stopTaosd { + echo "Stop taosd" + sudo systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +function dohavecore(){ + corefile=`find $corepath -mmin 1` + if [ -n "$corefile" ];then + echo 'taosd or taos has generated core' + if [[ $1 == 1 ]];then + exit 8 + fi + fi +} function runSimCaseOneByOne { while read -r line; do if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then - case=`echo $line | grep sim$ |awk '{print $NF}'` - - start_time=`date +%s` - ./test.sh -f $case > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ - echo -e "${RED}$case failed${NC}" | tee -a out.log + case=`echo $line | grep sim$ |awk '{print $NF}'` + start_time=`date +%s` + date +%F\ %T | tee -a out.log + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + echo -n $case + ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + echo -e "${RED} failed${NC}" | tee -a out.log + else + echo -n $case + ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + echo -e "${RED} failed${NC}" | tee -a out.log + fi out_log=`tail -1 out.log ` # if [[ $out_log =~ 'failed' ]];then # exit 8 # fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log + dohavecore 0 + fi + done < $1 +} +function runSimCaseOneByOnefq { + while read -r line; do + if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then + case=`echo $line | grep sim$ |awk '{print $NF}'` + + start_time=`date +%s` + date +%F\ %T | tee -a out.log + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + echo -n $case + ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + echo -e "${RED} failed${NC}" | tee -a out.log + else + echo -n $case + ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + echo -e "${RED} failed${NC}" | tee -a out.log + fi + + out_log=`tail -1 out.log ` + if [[ $out_log =~ 'failed' ]];then + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` + else + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + exit 8 + fi + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log + dohavecore 1 fi done < $1 } @@ -37,9 +108,11 @@ function runPyCaseOneByOne { case=`echo $line|awk '{print $NF}'` fi start_time=`date +%s` + date +%F\ %T | tee -a pytest-out.log + echo -n $case $line > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ - echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log + echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ + echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` # if [[ $out_log =~ 'failed' ]];then @@ -49,16 +122,48 @@ function runPyCaseOneByOne { else $line > /dev/null 2>&1 fi + dohavecore 0 + fi + done < $1 +} +function runPyCaseOneByOnefq { + while read -r line; do + if [[ $line =~ ^python.* ]]; then + if [[ $line != *sleep* ]]; then + + if [[ $line =~ '-r' ]];then + case=`echo $line|awk '{print $4}'` + else + case=`echo $line|awk '{print $NF}'` + fi + start_time=`date +%s` + date +%F\ %T | tee -a pytest-out.log + echo -n $case + $line > /dev/null 2>&1 && \ + echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ + echo -e "${RED} failed${NC}" | tee -a pytest-out.log + end_time=`date +%s` + out_log=`tail -1 pytest-out.log ` + if [[ $out_log =~ 'failed' ]];then + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + exit 8 + fi + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log + else + $line > /dev/null 2>&1 + fi + dohavecore 1 fi done < $1 } totalFailed=0 totalPyFailed=0 +totalJDBCFailed=0 +totalUnitFailed=0 -tests_dir=`pwd` - -if [ "$2" != "python" ]; then +corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` +if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]; then echo "### run TSIM test case ###" cd $tests_dir/script @@ -72,12 +177,37 @@ if [ "$2" != "python" ]; then elif [ "$1" == "b1" ]; then echo "### run TSIM b1 test ###" runSimCaseOneByOne jenkins/basic_1.txt + runSimCaseOneByOne jenkins/basic_4.txt + runSimCaseOneByOne jenkins/basic_5.txt + runSimCaseOneByOne jenkins/basic_6.txt + runSimCaseOneByOne jenkins/basic_7.txt elif [ "$1" == "b2" ]; then echo "### run TSIM b2 test ###" runSimCaseOneByOne jenkins/basic_2.txt elif [ "$1" == "b3" ]; then echo "### run TSIM b3 test ###" runSimCaseOneByOne jenkins/basic_3.txt + elif [ "$1" == "b1fq" ]; then + echo "### run TSIM b1 test ###" + runSimCaseOneByOnefq jenkins/basic_1.txt + elif [ "$1" == "b2fq" ]; then + echo "### run TSIM b2 test ###" + runSimCaseOneByOnefq jenkins/basic_2.txt + elif [ "$1" == "b3fq" ]; then + echo "### run TSIM b3 test ###" + runSimCaseOneByOnefq jenkins/basic_3.txt + elif [ "$1" == "b4fq" ]; then + echo "### run TSIM b4 test ###" + runSimCaseOneByOnefq jenkins/basic_4.txt + elif [ "$1" == "b5fq" ]; then + echo "### run TSIM b5 test ###" + runSimCaseOneByOnefq jenkins/basic_5.txt + elif [ "$1" == "b6fq" ]; then + echo "### run TSIM b6 test ###" + runSimCaseOneByOnefq jenkins/basic_6.txt + elif [ "$1" == "b7fq" ]; then + echo "### run TSIM b7 test ###" + runSimCaseOneByOnefq jenkins/basic_7.txt elif [ "$1" == "smoke" ] || [ -z "$1" ]; then echo "### run TSIM smoke test ###" runSimCaseOneByOne basicSuite.sim @@ -102,11 +232,10 @@ if [ "$2" != "python" ]; then fi fi -if [ "$2" != "sim" ]; then +if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]; then echo "### run Python test case ###" cd $tests_dir - IN_TDINTERNAL="community" if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then cd ../.. @@ -137,12 +266,21 @@ if [ "$2" != "sim" ]; then elif [ "$1" == "pytest" ]; then echo "### run Python full test ###" runPyCaseOneByOne fulltest.sh + elif [ "$1" == "pytestfq" ]; then + echo "### run Python full test ###" + runPyCaseOneByOnefq fulltest.sh elif [ "$1" == "p1" ]; then echo "### run Python_1 test ###" - runPyCaseOneByOne pytest_1.sh + runPyCaseOneByOnefq pytest_1.sh elif [ "$1" == "p2" ]; then echo "### run Python_2 test ###" - runPyCaseOneByOne pytest_2.sh + runPyCaseOneByOnefq pytest_2.sh + elif [ "$1" == "p3" ]; then + echo "### run Python_3 test ###" + runPyCaseOneByOnefq pytest_3.sh + elif [ "$1" == "p4" ]; then + echo "### run Python_4 test ###" + runPyCaseOneByOnefq pytest_4.sh elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then exit $(($totalFailed + $totalPyFailed)) elif [ "$1" == "smoke" ] || [ -z "$1" ]; then @@ -162,4 +300,84 @@ if [ "$2" != "sim" ]; then fi fi -exit $(($totalFailed + $totalPyFailed)) + +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then + echo "### run JDBC test cases ###" + + cd $tests_dir + + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../../ + else + cd ../ + fi + + pwd + cd debug/ + + stopTaosd + nohup build/bin/taosd -c /etc/taos/ > /dev/null 2>&1 & + sleep 30 + + cd $tests_dir/../src/connector/jdbc + + mvn test > jdbc-out.log 2>&1 + tail -n 20 jdbc-out.log + + cases=`grep 'Tests run' jdbc-out.log | awk 'END{print $3}'` + totalJDBCCases=`echo ${cases/%,}` + failed=`grep 'Tests run' jdbc-out.log | awk 'END{print $5}'` + JDBCFailed=`echo ${failed/%,}` + error=`grep 'Tests run' jdbc-out.log | awk 'END{print $7}'` + JDBCError=`echo ${error/%,}` + + totalJDBCFailed=`expr $JDBCFailed + $JDBCError` + totalJDBCSuccess=`expr $totalJDBCCases - $totalJDBCFailed` + + if [ "$totalJDBCSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalJDBCSuccess JDBC case(s) succeed! ### ${NC}" + fi + + if [ "$totalJDBCFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalJDBCFailed JDBC case(s) failed! ### ${NC}" + fi + dohavecore 1 +fi + +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$1" == "full" ]; then + echo "### run Unit tests ###" + + stopTaosd + cd $tests_dir + + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../../ + else + cd ../ + fi + + pwd + cd debug/build/bin + nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & + sleep 30 + + pwd + ./queryTest > unittest-out.log 2>&1 + tail -n 20 unittest-out.log + + totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'` + totalUnitSuccess=`grep 'PASSED' unittest-out.log | awk '{print $4}'` + totalUnitFailed=`expr $totalUnitTests - $totalUnitSuccess` + + if [ "$totalUnitSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalUnitSuccess Unit test succeed! ### ${NC}" + fi + + if [ "$totalUnitFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalUnitFailed Unit test failed! ### ${NC}" + fi + dohavecore 1 +fi + + +exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed + $totalUnitFailed)) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 11480a8ba2f7c2a78b8628e3edabe6e0eb122410..2eb8ee1614b286f3827705865cf073a7eded0c88 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -31,8 +31,8 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - add_executable(createNormalTable createNormalTable.c) - target_link_libraries(createNormalTable taos_static tutil common pthread) + #add_executable(createNormalTable createNormalTable.c) + #target_link_libraries(createNormalTable taos_static tutil common pthread) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) @@ -46,7 +46,7 @@ IF (TD_LINUX) #add_executable(invalidTableId invalidTableId.c) #target_link_libraries(invalidTableId taos_static tutil common pthread) - add_executable(hashIterator hashIterator.c) - target_link_libraries(hashIterator taos_static tutil common pthread) + #add_executable(hashIterator hashIterator.c) + #target_link_libraries(hashIterator taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/createTablePerformance.c b/tests/test/c/createTablePerformance.c index eae104291a3bfcbb37e6a2c89d61214c61d4b6c0..b94c687f2cba7310949b0a3b12b6f4fc007e5a9a 100644 --- a/tests/test/c/createTablePerformance.c +++ b/tests/test/c/createTablePerformance.c @@ -32,6 +32,7 @@ int32_t numOfThreads = 30; int32_t numOfTables = 100000; int32_t replica = 1; int32_t numOfColumns = 2; +TAOS * con = NULL; typedef struct { int32_t tableBeginIndex; @@ -84,13 +85,14 @@ int main(int argc, char *argv[]) { pthread_attr_destroy(&thattr); free(pInfo); + taos_close(con); } void createDbAndSTable() { pPrint("start to create db and stable"); char qstr[64000]; - TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { pError("failed to connect to DB, reason:%s", taos_errstr(con)); exit(1); @@ -127,8 +129,6 @@ void createDbAndSTable() { exit(0); } taos_free_result(pSql); - - taos_close(con); } void *threadFunc(void *param) { @@ -136,12 +136,6 @@ void *threadFunc(void *param) { char qstr[65000]; int code; - TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (con == NULL) { - pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); - exit(1); - } - sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pSql = taos_query(con, qstr); taos_free_result(pSql); @@ -170,7 +164,6 @@ void *threadFunc(void *param) { pInfo->createTableSpeed = speed; pPrint("thread:%d, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, seconds, speed); - taos_close(con); return 0; } diff --git a/tests/test/c/hashPerformance.c b/tests/test/c/hashPerformance.c index db3be0e950a81e22f3ad1050cfa9a8e50b967532..111ea25a09823fbbeeda3c990d86af849c3821d8 100644 --- a/tests/test/c/hashPerformance.c +++ b/tests/test/c/hashPerformance.c @@ -24,15 +24,17 @@ #define GREEN "\033[1;32m" #define NC "\033[0m" -int32_t capacity = 100000; -int32_t q1Times = 1; -int32_t q2Times = 1; +int32_t capacity = 128; +int32_t q1Times = 10; +int32_t q2Times = 10; int32_t keyNum = 100000; -int32_t printInterval = 10000; +int32_t printInterval = 1000; +void * hashHandle; +pthread_t thread; typedef struct HashTestRow { - int32_t size; - void * ptr; + int32_t keySize; + char key[100]; } HashTestRow; void shellParseArgument(int argc, char *argv[]); @@ -40,7 +42,7 @@ void shellParseArgument(int argc, char *argv[]); void testHashPerformance() { int64_t initialMs = taosGetTimestampMs(); _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - void * hashHandle = taosHashInit(capacity, hashFp, true); + hashHandle = taosHashInit(128, hashFp, true, HASH_NO_LOCK); int64_t startMs = taosGetTimestampMs(); float seconds = (startMs - initialMs) / 1000.0; @@ -48,17 +50,25 @@ void testHashPerformance() { for (int32_t t = 1; t <= keyNum; ++t) { HashTestRow row = {0}; - char key[100] = {0}; - int32_t keySize = sprintf(key, "0.db.st%d", t); + row.keySize = sprintf(row.key, "0.db.st%d", t); for (int32_t q = 0; q < q1Times; q++) { - taosHashGet(hashHandle, &key, keySize); + taosHashGet(hashHandle, row.key, row.keySize); } - taosHashPut(hashHandle, key, keySize, &row, sizeof(HashTestRow)); + taosHashPut(hashHandle, row.key, row.keySize, &row, sizeof(HashTestRow)); for (int32_t q = 0; q < q2Times; q++) { - taosHashGet(hashHandle, &key, keySize); + taosHashGet(hashHandle, row.key, row.keySize); + } + + // test iterator + { + HashTestRow *row = taosHashIterate(hashHandle, NULL); + while (row) { + taosHashGet(hashHandle, row->key, row->keySize); + row = taosHashIterate(hashHandle, row); + } } if (t % printInterval == 0) { @@ -80,9 +90,35 @@ void testHashPerformance() { taosHashCleanup(hashHandle); } +void *multiThreadFunc(void *param) { + for (int i = 0; i < 100; ++i) { + taosMsleep(1000); + HashTestRow *row = taosHashIterate(hashHandle, NULL); + while (row) { + taosHashGet(hashHandle, row->key, row->keySize); + row = taosHashIterate(hashHandle, row); + } + int64_t hashSize = taosHashGetSize(hashHandle); + pPrint("i:%d hashSize:%ld", i, hashSize); + } + + return NULL; +} + +void multiThreadTest() { + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + // Start threads to write + pthread_create(&thread, &thattr, multiThreadFunc, NULL); +} + int main(int argc, char *argv[]) { shellParseArgument(argc, argv); + multiThreadTest(); testHashPerformance(); + pthread_join(thread, NULL); } void printHelp() { diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 2db750cdd302c63522e008e3ea324230dd87b6e5..dbda5f1bbdd01a451376f8afe7717b7290980796 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -146,7 +146,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { int32_t simExecuteExpression(SScript *script, char *exp) { char * op1, *op2, *var1, *var2, *var3, *rest; int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; - char t0[512], t1[512], t2[512], t3[1024]; + char t0[1024], t1[1024], t2[1024], t3[2048]; int32_t result; rest = paGetToken(exp, &var1, &var1Len); @@ -292,17 +292,46 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { if (pthread_create(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); return false; + } else { + simDebug("script:%s, background thread:0x%08" PRIx64 " is created", newScript->fileName, + taosGetPthreadId(newScript->bgPid)); } script->linePos++; return true; } +void simReplaceShToBat(char *dst) { + char* sh = strstr(dst, ".sh"); + if (sh != NULL) { + int32_t dstLen = (int32_t)strlen(dst); + char *end = dst + dstLen; + *(end + 1) = 0; + + for (char *p = end; p >= sh; p--) { + *(p + 1) = *p; + } + + sh[0] = '.'; + sh[1] = 'b'; + sh[2] = 'a'; + sh[3] = 't'; + sh[4] = ' '; + } + + simDebug("system cmd is %s", dst); +} + bool simExecuteSystemCmd(SScript *script, char *option) { char buf[4096] = {0}; +#ifndef WINDOWS sprintf(buf, "cd %s; ", tsScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); +#else + sprintf(buf, "%s%s", tsScriptDir, option); + simReplaceShToBat(buf); +#endif simLogSql(buf, true); int32_t code = system(buf); @@ -311,9 +340,7 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, errno, strerror(errno), repeatTimes); taosMsleep(1000); -#ifdef LINUX - signal(SIGCHLD, SIG_DFL); -#endif + taosDflSignal(SIGCHLD); if (repeatTimes++ >= 10) { exit(0); } @@ -448,7 +475,6 @@ void simCloseNativeConnect(SScript *script) { simDebug("script:%s, taos:%p closed", script->fileName, script->taos); taos_close(script->taos); - taosMsleep(1200); script->taos = NULL; } @@ -744,15 +770,27 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { case TSDB_DATA_TYPE_TINYINT: sprintf(value, "%d", *((int8_t *)row[i])); break; + case TSDB_DATA_TYPE_UTINYINT: + sprintf(value, "%u", *((uint8_t*)row[i])); + break; case TSDB_DATA_TYPE_SMALLINT: sprintf(value, "%d", *((int16_t *)row[i])); break; + case TSDB_DATA_TYPE_USMALLINT: + sprintf(value, "%u", *((uint16_t *)row[i])); + break; case TSDB_DATA_TYPE_INT: sprintf(value, "%d", *((int32_t *)row[i])); break; + case TSDB_DATA_TYPE_UINT: + sprintf(value, "%u", *((uint32_t *)row[i])); + break; case TSDB_DATA_TYPE_BIGINT: sprintf(value, "%" PRId64, *((int64_t *)row[i])); break; + case TSDB_DATA_TYPE_UBIGINT: + sprintf(value, "%" PRIu64, *((uint64_t *)row[i])); + break; case TSDB_DATA_TYPE_FLOAT: sprintf(value, "%.5f", GET_FLOAT_VAL(row[i])); break; diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 33fd24dd5823b8ec7bf818cec7c36ddac783bc24..990ea7141335b3971b7a3b4dca5fc6fec6dcd24f 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -20,8 +20,9 @@ #undef TAOS_MEM_CHECK bool simAsyncQuery = false; +bool simExecSuccess = false; -void simHandleSignal(int32_t signo) { +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); exit(1); } @@ -40,27 +41,30 @@ int32_t main(int32_t argc, char *argv[]) { printf("usage: %s [options] \n", argv[0]); printf(" [-c config]: config directory, default is: %s\n", configDir); printf(" [-f script]: script filename\n"); - exit(0); + return 0; } } if (!simSystemInit()) { simError("failed to initialize the system"); simSystemCleanUp(); - exit(1); + return -1; } simInfo("simulator is running ..."); - signal(SIGINT, simHandleSignal); + taosSetSignal(SIGINT, simHandleSignal); SScript *script = simParseScript(scriptFile); if (script == NULL) { simError("parse script file:%s failed", scriptFile); - exit(-1); + return -1; } simScriptList[++simScriptPos] = script; simExecuteScript(script); - return 0; + int32_t ret = simExecSuccess ? 0 : -1; + simInfo("execute result %d", ret); + + return ret; } \ No newline at end of file diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 693ade7b35a095b499198db9dcc27335f73e14bd..40937e70532897fa6776965b861a8d678532d8aa 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -21,6 +21,7 @@ #include "ttimer.h" #include "tutil.h" #include "tsocket.h" +#include "taoserror.h" #undef TAOS_MEM_CHECK SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; @@ -31,6 +32,8 @@ int32_t simDebugFlag = 135; void simCloseTaosdConnect(SScript *script); char simHostName[128]; +extern bool simExecSuccess; + char *simParseArbitratorName(char *varName) { static char hostName[140]; sprintf(hostName, "%s:%d", simHostName, 8000); @@ -78,8 +81,10 @@ char *simParseHostName(char *varName) { } bool simSystemInit() { + if (taos_init()) { + return false; + } taosGetFqdn(simHostName); - taos_init(); simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); return true; @@ -93,28 +98,37 @@ void simFreeScript(SScript *script) { for (int32_t i = 0; i < script->bgScriptLen; ++i) { SScript *bgScript = script->bgScripts[i]; - simInfo("script:%s, set stop flag", script->fileName); + simDebug("script:%s, is background script, set stop flag", bgScript->fileName); bgScript->killed = true; if (taosCheckPthreadValid(bgScript->bgPid)) { pthread_join(bgScript->bgPid, NULL); } + + simDebug("script:%s, background thread joined", bgScript->fileName); + taos_close(bgScript->taos); + tfree(bgScript->lines); + tfree(bgScript->optionBuffer); + tfree(bgScript); } - } - simDebug("script:%s, is freed", script->fileName); - taos_close(script->taos); - tfree(script->lines); - tfree(script->optionBuffer); - tfree(script); + simDebug("script:%s, is cleaned", script->fileName); + taos_close(script->taos); + tfree(script->lines); + tfree(script->optionBuffer); + tfree(script); + } } SScript *simProcessCallOver(SScript *script) { if (script->type == SIM_SCRIPT_TYPE_MAIN) { + simDebug("script:%s, is main script, set stop flag", script->fileName); if (script->killed) { + simExecSuccess = false; simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", script->fileName, script->error); - exit(-1); + return NULL; } else { + simExecSuccess = true; simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, script->fileName); simCloseTaosdConnect(script); @@ -125,13 +139,13 @@ SScript *simProcessCallOver(SScript *script) { if (simScriptPos == -1) { simInfo("----------------------------------------------------------------------"); simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); - exit(0); + return NULL; } return simScriptList[simScriptPos]; } } else { - simInfo("script:%s, is stopped by main script", script->fileName); + simDebug("script:%s, is stopped", script->fileName); simFreeScript(script); return NULL; } @@ -161,5 +175,6 @@ void *simExecuteScript(void *inputScript) { } } + simInfo("thread is stopped"); return NULL; }