diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index e7802b3d0d3cf969381b5fa2a99862b3df5aa05f..0000000000000000000000000000000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 1.0.{build} -image: - - Visual Studio 2015 - - macos -environment: - matrix: - - ARCH: amd64 - - ARCH: x86 -matrix: - exclude: - - image: macos - ARCH: x86 -for: - - - matrix: - only: - - image: Visual Studio 2015 - clone_folder: c:\dev\TDengine - clone_depth: 1 - - init: - - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %ARCH% - - before_build: - - cd c:\dev\TDengine - - md build - - build_script: - - cd build - - cmake -G "NMake Makefiles" .. -DBUILD_JDBC=false - - nmake install - - - matrix: - only: - - image: macos - clone_depth: 1 - - build_script: - - mkdir debug - - cd debug - - cmake .. > /dev/null - - make > /dev/null -notifications: -- provider: Email - to: - - sangshuduo@gmail.com - on_build_success: true - on_build_failure: true - on_build_status_changed: true diff --git a/.gitignore b/.gitignore index 50f4251320abc80358b67eab22c02672d5f26bd6..0e7125c6565e0809b39098502589758aa6cb02de 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,10 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ +tests/script/api/batchprepare +tests/script/api/stmt +tests/script/api/stmtBatchTest +tests/script/api/stmtTest # Emacs # -*- mode: gitignore; -*- diff --git a/Jenkinsfile b/Jenkinsfile index ff470edce37c4a7ea4e545be6b3d2f3672d83340..e96455f20d01593b22064b005b0b6be69b89a76a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,7 @@ import hudson.model.Result import hudson.model.*; import jenkins.model.CauseOfInterruption -properties([pipelineTriggers([githubPush()])]) node { - git url: 'https://github.com/taosdata/TDengine.git' } def skipbuild=0 @@ -40,7 +38,8 @@ def pre_test(){ sudo rmtaos || echo "taosd has not installed" ''' sh ''' - killall -9 taosd ||echo "no taosd running" + kill -9 $(pidof taosd) ||echo "no taosd running" + kill -9 $(pidof taosadapter) ||echo "no taosadapter running" killall -9 gdb || echo "no gdb running" killall -9 python3.8 || echo "no python program running" cd ${WKC} @@ -72,6 +71,7 @@ def pre_test(){ git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git clean -dfx + git submodule update --init --recursive cd ${WK} git reset --hard HEAD~10 ''' @@ -112,8 +112,149 @@ def pre_test(){ ''' return 1 } +def pre_test_noinstall(){ + sh'hostname' + sh''' + cd ${WKC} + git reset --hard HEAD~10 >/dev/null + ''' + script { + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } + else{ + sh ''' + cd ${WKC} + git checkout develop + ''' + } + } + sh''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git clean -dfx + git submodule update --init --recursive + cd ${WK} + git reset --hard HEAD~10 + ''' + script { + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WK} + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WK} + git checkout 2.0 + ''' + } + else{ + sh ''' + cd ${WK} + git checkout develop + ''' + } + } + sh ''' + cd ${WK} + git pull >/dev/null + + export TZ=Asia/Harbin + date + git clean -dfx + mkdir debug + cd debug + cmake .. > /dev/null + make + ''' + return 1 +} +def pre_test_ningsi(){ + sh'hostname' + sh''' + cd ${WKC} + git reset --hard HEAD~10 >/dev/null + ''' + script { + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } + else{ + sh ''' + cd ${WKC} + git checkout develop + ''' + } + } + sh''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git clean -dfx + git submodule update --init --recursive + cd ${WK} + git reset --hard HEAD~10 + ''' + script { + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WK} + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WK} + git checkout 2.0 + ''' + } + else{ + sh ''' + cd ${WK} + git checkout develop + ''' + } + } + sh ''' + cd ${WK} + git pull >/dev/null + + export TZ=Asia/Harbin + date + git clean -dfx + mkdir debug + cd debug + cmake .. -DOSTYPE=Ningsi60 > /dev/null + make + ''' + return 1 +} def pre_test_win(){ bat ''' + taskkill /f /t /im python.exe cd C:\\ rd /s /Q C:\\TDengine cd C:\\workspace\\TDinternal @@ -147,6 +288,7 @@ def pre_test_win(){ git fetch origin +refs/pull/%CHANGE_ID%/merge git checkout -qf FETCH_HEAD git clean -dfx + git submodule update --init --recursive cd C:\\workspace\\TDinternal git reset --hard HEAD~10 ''' @@ -171,7 +313,6 @@ def pre_test_win(){ } } bat ''' - taskkill /f /t /im python.exe cd C:\\workspace\\TDinternal git pull @@ -179,9 +320,9 @@ def pre_test_win(){ git clean -dfx mkdir debug cd debug - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64 + call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" amd64 cmake ../ -G "NMake Makefiles" - nmake || exit 8 + set CL=/MP nmake nmake || exit 8 nmake install || exit 8 xcopy /e/y/i/f C:\\workspace\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 || exit 8 cd C:\\workspace\\TDinternal\\community\\src\\connector\\python @@ -192,6 +333,7 @@ def pre_test_win(){ } pipeline { agent none + options { skipDefaultCheckout() } environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDinternal/community' @@ -199,6 +341,7 @@ pipeline { stages { stage('pre_build'){ agent{label 'master'} + options { skipDefaultCheckout() } when { changeRequest() } @@ -207,52 +350,52 @@ pipeline { abort_previous() abortPreviousBuilds() } - sh''' - rm -rf ${WORKSPACE}.tes - cp -r ${WORKSPACE} ${WORKSPACE}.tes - cd ${WORKSPACE}.tes - git fetch - ''' - script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - git checkout 2.0 - ''' - } - else{ - sh ''' - git checkout develop - ''' - } - } - sh''' - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD - ''' + // sh''' + // rm -rf ${WORKSPACE}.tes + // cp -r ${WORKSPACE} ${WORKSPACE}.tes + // cd ${WORKSPACE}.tes + // git fetch + // ''' + // script { + // if (env.CHANGE_TARGET == 'master') { + // sh ''' + // git checkout master + // ''' + // } + // else if(env.CHANGE_TARGET == '2.0'){ + // sh ''' + // git checkout 2.0 + // ''' + // } + // else{ + // sh ''' + // git checkout develop + // ''' + // } + // } + // sh''' + // git fetch origin +refs/pull/${CHANGE_ID}/merge + // git checkout -qf FETCH_HEAD + // ''' - script{ - skipbuild='2' - skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) - println skipbuild - } - sh''' - rm -rf ${WORKSPACE}.tes - ''' + // script{ + // skipbuild='2' + // skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) + // println skipbuild + // } + // sh''' + // rm -rf ${WORKSPACE}.tes + // ''' + // } } } stage('Parallel test stage') { //only build pr + options { skipDefaultCheckout() } when { allOf{ changeRequest() - expression{ - return skipbuild.trim() == '2' - } + not{ expression { env.CHANGE_BRANCH =~ /docs\// }} } } parallel { @@ -288,7 +431,7 @@ pipeline { stage('python_3_s6') { agent{label " slave6 || slave16 "} steps { - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 65, unit: 'MINUTES'){ pre_test() sh ''' date @@ -314,11 +457,13 @@ pipeline { npm install td2.0-connector > /dev/null 2>&1 node nodejsChecker.js host=localhost ''' - sh ''' - cd ${WKC}/tests/examples/C#/taosdemo - mcs -out:taosdemo *.cs > /dev/null 2>&1 - ./taosdemo -c /etc/taos -y - ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/C#/taosdemo + mcs -out:taosdemo *.cs > /dev/null 2>&1 + echo '' |./taosdemo -c /etc/taos + ''' + } sh ''' cd ${WKC}/tests/gotest bash batchtest.sh @@ -394,7 +539,7 @@ pipeline { stage('test_b4_s7') { agent{label " slave7 || slave17 "} steps { - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 105, unit: 'MINUTES'){ pre_test() sh ''' date @@ -426,7 +571,7 @@ pipeline { stage('test_b6_s9') { agent{label " slave9 || slave19 "} steps { - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 105, unit: 'MINUTES'){ pre_test() sh ''' date @@ -449,36 +594,97 @@ pipeline { } } } + stage('arm64centos7') { + agent{label " arm64centos7 "} + steps { + pre_test_noinstall() + } + } + stage('arm64centos8') { + agent{label " arm64centos8 "} + steps { + pre_test_noinstall() + } + } + stage('arm32bionic') { + agent{label " arm32bionic "} + steps { + pre_test_noinstall() + } + } + stage('arm64bionic') { + agent{label " arm64bionic "} + steps { + pre_test_noinstall() + } + } + stage('arm64focal') { + agent{label " arm64focal "} + steps { + pre_test_noinstall() + } + } + stage('centos7') { + agent{label " centos7 "} + steps { + pre_test_noinstall() + } + } + stage('ubuntu:trusty') { + agent{label " trusty "} + steps { + pre_test_noinstall() + } + } + stage('ubuntu:xenial') { + agent{label " xenial "} + steps { + pre_test_noinstall() + } + } + stage('ubuntu:bionic') { + agent{label " bionic "} + steps { + pre_test_noinstall() + } + } + + stage('ningsi') { + agent{label "ningsi"} + steps { + pre_test_ningsi() + } + } - // stage('build'){ - // agent{label " wintest "} - // steps { - // pre_test() - // script{ - // while(win_stop == 0){ - // sleep(1) - // } - // } - // } - // } - // stage('test'){ - // agent{label "win"} - // steps{ + stage('build'){ + agent{label " wintest "} + steps { + pre_test() + script{ + while(win_stop == 0){ + sleep(1) + } + } + } + } + stage('test'){ + agent{label "win"} + steps{ - // catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - // pre_test_win() - // timeout(time: 20, unit: 'MINUTES'){ - // bat''' - // cd C:\\workspace\\TDinternal\\community\\tests\\pytest - // .\\test-all.bat Wintest - // ''' - // } - // } - // script{ - // win_stop=1 - // } - // } - // } + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + pre_test_win() + timeout(time: 20, unit: 'MINUTES'){ + bat''' + cd C:\\workspace\\TDinternal\\community\\tests\\pytest + .\\test-all.bat wintest + ''' + } + } + script{ + win_stop=1 + } + } + } } diff --git a/cmake/define.inc b/cmake/define.inc index 87c88b35a9e9f68ce9d30e340f5b13570ce00231..f29f9c4cd8e2314a0b6f1dd5e18d9ecdaed9f59e 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -49,6 +49,14 @@ IF (TD_PRO) ADD_DEFINITIONS(-D_TD_PRO_) ENDIF () +IF (TD_KH) + ADD_DEFINITIONS(-D_TD_KH_) +ENDIF () + +IF (TD_JH) + ADD_DEFINITIONS(-D_TD_JH_) +ENDIF () + IF (TD_MEM_CHECK) ADD_DEFINITIONS(-DTAOS_MEM_CHECK) ENDIF () @@ -117,7 +125,7 @@ IF (TD_MIPS_32) SET(COMMON_FLAGS "-Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () -IF (TD_APLHINE) +IF (TD_ALPINE) SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") link_libraries(/usr/lib/libargp.a) ADD_DEFINITIONS(-D_ALPINE) @@ -136,7 +144,12 @@ IF (TD_LINUX) ENDIF () IF (TD_MEMORY_SANITIZER) - SET(DEBUG_FLAGS "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -O0 -g3 -DDEBUG") + IF (TD_ARCHLINUX) + SET(DEBUG_FLAGS "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -O0 -g3 -DDEBUG") + ELSE () + SET(DEBUG_FLAGS "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -O0 -g3 -DDEBUG") + ENDIF () + MESSAGE(STATUS "${BoldRed}Will compile with memory sanitizer! ${ColourReset}") ELSE () SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") ENDIF () diff --git a/cmake/input.inc b/cmake/input.inc index d746cf52f6eb016795d6fa6d01f408925159c710..bff76a567973633fd4d7c3df5dae4b1a8c334d39 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -52,6 +52,12 @@ ELSEIF (${DBNAME} MATCHES "tq") ELSEIF (${DBNAME} MATCHES "pro") SET(TD_PRO TRUE) MESSAGE(STATUS "pro is true") +ELSEIF (${DBNAME} MATCHES "kh") + SET(TD_KH TRUE) + MESSAGE(STATUS "kh is true") +ELSEIF (${DBNAME} MATCHES "jh") + SET(TD_JH TRUE) + MESSAGE(STATUS "jh is true") ENDIF () IF (${DLLTYPE} MATCHES "go") diff --git a/cmake/install.inc b/cmake/install.inc index 5a39173697e596cb9fa96a883000d2f3ea996c46..48d594e8898b5a278dc46fbec1e61ebe90ead905 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -1,23 +1,22 @@ IF (TD_LINUX) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") - INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})") + INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})") ELSEIF (TD_WINDOWS) IF (TD_POWER) SET(CMAKE_INSTALL_PREFIX C:/PowerDB) + ELSEIF (TD_TQ) + SET(CMAKE_INSTALL_PREFIX C:/TQueue) ELSEIF (TD_PRO) SET(CMAKE_INSTALL_PREFIX C:/ProDB) + ELSEIF (TD_KH) + SET(CMAKE_INSTALL_PREFIX C:/KingHistorian) + ELSEIF (TD_JH) + SET(CMAKE_INSTALL_PREFIX C:/jh_iot) ELSE () SET(CMAKE_INSTALL_PREFIX C:/TDengine) ENDIF () - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taoserror.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) @@ -25,10 +24,28 @@ ELSEIF (TD_WINDOWS) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) IF (TD_POWER) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/power.exe DESTINATION .) + ELSEIF (TD_TQ) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/tq.exe DESTINATION .) ELSEIF (TD_PRO) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/prodbc.exe DESTINATION .) + ELSEIF (TD_KH) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/kinghistorian.cfg DESTINATION cfg) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/khclient.exe DESTINATION .) + ELSEIF (TD_JH) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/jh_taos.exe DESTINATION .) ELSE () + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) + + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosdemo.exe DESTINATION .) ENDIF () @@ -41,6 +58,5 @@ ELSEIF (TD_WINDOWS) ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") - INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin ${TD_VER_NUMBER})") + INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin ${TD_VER_NUMBER})") ENDIF () diff --git a/cmake/platform.inc b/cmake/platform.inc index a78082a1fc62a8ad66c54dcf005e3e15edf5f5f0..9ef04815c3150cf18193895475c93389cf26aac2 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -21,7 +21,7 @@ SET(TD_LINUX FALSE) SET(TD_ARM_32 FALSE) SET(TD_MIPS_64 FALSE) SET(TD_MIPS_32 FALSE) - SET(TD_APLHINE FALSE) + SET(TD_ALPINE FALSE) SET(TD_NINGSI FALSE) SET(TD_NINGSI_60 FALSE) SET(TD_NINGSI_80 FALSE) @@ -36,7 +36,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") # Get OS information and store in variable TD_OS_INFO. # execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh) - execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO) + execute_process(COMMAND sh ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO) MESSAGE(STATUS "The current os is " ${TD_OS_INFO}) SET(TD_LINUX TRUE) @@ -52,8 +52,13 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") ENDIF () IF (${TD_OS_INFO} MATCHES "Alpine") - SET(TD_APLHINE TRUE) - MESSAGE(STATUS "The current OS is Alpine, append extra flags") + SET(TD_ALPINE TRUE) + MESSAGE(STATUS "The current OS is Alpine Linux, append extra flags") + ELSEIF (${TD_OS_INFO} MATCHES "Arch") + SET(TD_ARCHLINUX TRUE) + MESSAGE(STATUS "The current OS is Arch Linux") + ELSE () + MESSAGE(STATUS "Ths distro is " ${TD_OS_INFO}) ENDIF() ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") SET(TD_DARWIN TRUE) @@ -155,7 +160,7 @@ ELSEIF (${OSTYPE} MATCHES "Linux") MESSAGE(STATUS "input osType: Linux") ELSEIF (${OSTYPE} MATCHES "Alpine") MESSAGE(STATUS "input osType: Alpine") - SET(TD_APLHINE TRUE) + SET(TD_ALPINE TRUE) ELSE () MESSAGE(STATUS "The user specified osType is unknown: " ${OSTYPE}) ENDIF () diff --git a/cmake/version.inc b/cmake/version.inc index 14dd11cd26edca7b5a28efd9e53ebed32b927658..20141827d93cfb61445659b846130cbb35816040 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.2.1.0") + SET(TD_VER_NUMBER "2.2.2.4") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/deps/MsvcLibX/src/filetime.c b/deps/MsvcLibX/src/filetime.c index 084b5eef671c0cc17135746bd7d7e7fb6ed68156..27d4c2ea46f6370e8f4fa7f651a0b4b6e6f2ea2c 100644 --- a/deps/MsvcLibX/src/filetime.c +++ b/deps/MsvcLibX/src/filetime.c @@ -29,7 +29,7 @@ A Unix time_t is the number of 1-second intervals since January 1, 1970. time_ts are expressed in the GMT time zone. DOS times in the current local time. */ -time_t Filetime2Timet(uint16_t date, uint16_t time) { +time_t Filetime2Timet(uint16_t date, uint16_t deps_time) { unsigned int year, month, day, hour, minute, second; struct tm stm; @@ -37,9 +37,9 @@ time_t Filetime2Timet(uint16_t date, uint16_t time) { year = 1980 + ((date & 0xFE00) >> 9); month = (date & 0x1E0) >> 5; day = date & 0x1F; - hour = (time & 0xF800) >> 11; - minute = (time & 0x7E0) >> 5; - second = 2 * (time & 0x1F); + hour = (deps_time & 0xF800) >> 11; + minute = (deps_time & 0x7E0) >> 5; + second = 2 * (deps_time & 0x1F); stm.tm_year = (int)year - 1900; stm.tm_mon = (int)month - 1; @@ -55,7 +55,7 @@ time_t Filetime2Timet(uint16_t date, uint16_t time) { #if 0 /* Older version of the same, trying to generate the time_t manually. Did not handle DST well */ -time_t Filetime2Timet(uint16_t date, uint16_t time) { +time_t Filetime2Timet(uint16_t date, uint16_t deps_time) { unsigned int year, month, day, hour, minute, second; unsigned int olympiads; /* 4-year periods */ unsigned long t = 0; @@ -64,9 +64,9 @@ time_t Filetime2Timet(uint16_t date, uint16_t time) { year = 1980 + ((date & 0xFE00) >> 9); month = (date & 0x1E0) >> 5; day = date & 0x1F; - hour = (time & 0xF800) >> 11; - minute = (time & 0x7E0) >> 5; - second = 2 * (time & 0x1F); + hour = (deps_time & 0xF800) >> 11; + minute = (deps_time & 0x7E0) >> 5; + second = 2 * (deps_time & 0x1F); /* Count days */ year -= 1970; /* Start of Unix time_t epoch */ @@ -111,16 +111,16 @@ time_t Filetime2Timet(uint16_t date, uint16_t time) { #endif /* Generate a string with the local file time, in the ISO 8601 date/time format */ -char *Filetime2String(uint16_t date, uint16_t time, char *pBuf, size_t nBufSize) { +char *Filetime2String(uint16_t date, uint16_t deps_time, char *pBuf, size_t nBufSize) { unsigned int year, month, day, hour, minute, second; /* Decode fields */ year = 1980 + ((date & 0xFE00) >> 9); month = (date & 0x1E0) >> 5; day = date & 0x1F; - hour = (time & 0xF800) >> 11; - minute = (time & 0x7E0) >> 5; - second = 2 * (time & 0x1F); + hour = (deps_time & 0xF800) >> 11; + minute = (deps_time & 0x7E0) >> 5; + second = 2 * (deps_time & 0x1F); if (nBufSize >= 20) { sprintf(pBuf, "%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); diff --git a/deps/cJson/src/cJSON.c b/deps/cJson/src/cJSON.c index f0ef9f6fe1715336ed8d24d4998df5a8ba51b3af..08d2c17adf2c4d9cc11198e2735b8ccbe8893ff6 100644 --- a/deps/cJson/src/cJSON.c +++ b/deps/cJson/src/cJSON.c @@ -49,8 +49,8 @@ typedef struct { const unsigned char *json; size_t position; -} error; -static error global_error = { NULL, 0 }; +} deps_error; +static deps_error global_error = { NULL, 0 }; CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) { @@ -210,8 +210,8 @@ typedef struct #define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) #define cannot_read(buffer, size) (!can_read(buffer, size)) /* check if the buffer can be accessed at the given index (starting with 0) */ -#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) -#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index)) +#define can_access_at_index(buffer, deps_index) ((buffer != NULL) && (((buffer)->offset + deps_index) < (buffer)->length)) +#define cannot_access_at_index(buffer, deps_index) (!can_access_at_index(buffer, deps_index)) /* get a pointer to the buffer at the position */ #define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) @@ -992,7 +992,7 @@ fail: if (value != NULL) { - error local_error; + deps_error local_error; local_error.json = (const unsigned char*)value; local_error.position = 0; @@ -1683,7 +1683,7 @@ CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) return (int)size; } -static cJSON* get_array_item(const cJSON *array, size_t index) +static cJSON* get_array_item(const cJSON *array, size_t deps_index) { cJSON *current_child = NULL; @@ -1693,23 +1693,23 @@ static cJSON* get_array_item(const cJSON *array, size_t index) } current_child = array->child; - while ((current_child != NULL) && (index > 0)) + while ((current_child != NULL) && (deps_index > 0)) { - index--; + deps_index--; current_child = current_child->next; } return current_child; } -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) +CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int deps_index) { - if (index < 0) + if (deps_index < 0) { return NULL; } - return get_array_item(array, (size_t)index); + return get_array_item(array, (size_t)deps_index); } static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive) diff --git a/deps/iconv/gentranslit.c b/deps/iconv/gentranslit.c index 200d0819fda46b27ebbbfdb98e8835098de8a4a7..a845013d1584a76532a67b51ae452bbfcdaf0eb1 100644 --- a/deps/iconv/gentranslit.c +++ b/deps/iconv/gentranslit.c @@ -30,7 +30,7 @@ int main (int argc, char *argv[]) { unsigned int data[0x100000]; int uni2index[0x110000]; - int index; + int deps_index; if (argc != 1) exit(1); @@ -64,7 +64,7 @@ int main (int argc, char *argv[]) int j; for (j = 0; j < 0x110000; j++) uni2index[j] = -1; - index = 0; + deps_index = 0; for (;;) { c = getc(stdin); if (c == EOF) @@ -86,8 +86,8 @@ int main (int argc, char *argv[]) if (c == '\t') break; if (uni2index[j] < 0) { - uni2index[j] = index; - data[index++] = 0; + uni2index[j] = deps_index; + data[deps_index++] = 0; } if (c >= 0x80) { /* Finish reading an UTF-8 character. */ @@ -104,17 +104,17 @@ int main (int argc, char *argv[]) } } } - data[index++] = (unsigned int) c; + data[deps_index++] = (unsigned int) c; } if (uni2index[j] >= 0) - data[uni2index[j]] = index - uni2index[j] - 1; + data[uni2index[j]] = deps_index - uni2index[j] - 1; do { c = getc(stdin); } while (!(c == EOF || c == '\n')); } } - printf("static const unsigned int translit_data[%d] = {",index); + printf("static const unsigned int translit_data[%d] = {",deps_index); { int i; - for (i = 0; i < index; i++) { + for (i = 0; i < deps_index; i++) { if (data[i] < 32) printf("\n %3d,",data[i]); else if (data[i] == '\'') diff --git a/deps/iconv/iconv.c b/deps/iconv/iconv.c index 391e35d4e78efda0cb881b247cbccf22ab66c67b..af5b32e112e3f3bf3e360c9e475736ffbbb4c004 100644 --- a/deps/iconv/iconv.c +++ b/deps/iconv/iconv.c @@ -491,7 +491,7 @@ const char * iconv_canonicalize (const char * name) char* bp; const struct alias * ap; unsigned int count; - unsigned int index; + unsigned int deps_index; const char* pool; /* Before calling aliases_lookup, convert the input string to upper case, @@ -555,23 +555,23 @@ const char * iconv_canonicalize (const char * name) This is also the case on native Woe32 systems. */ #if __STDC_ISO_10646__ || ((defined _WIN32 || defined __WIN32__) && !defined __CYGWIN__) if (sizeof(wchar_t) == 4) { - index = ei_ucs4internal; + deps_index = ei_ucs4internal; break; } if (sizeof(wchar_t) == 2) { - index = ei_ucs2internal; + deps_index = ei_ucs2internal; break; } if (sizeof(wchar_t) == 1) { - index = ei_iso8859_1; + deps_index = ei_iso8859_1; break; } #endif } - index = ap->encoding_index; + deps_index = ap->encoding_index; break; } - return all_canonical[index] + pool; + return all_canonical[deps_index] + pool; invalid: return name; } diff --git a/deps/jni/windows/win32/bridge/AccessBridgeCalls.c b/deps/jni/windows/win32/bridge/AccessBridgeCalls.c index 6ed699f34faca966aebaa5841f313b5a3262d2d7..4942508734569e1d6bf93b4eb8757a7804b0a8e0 100644 --- a/deps/jni/windows/win32/bridge/AccessBridgeCalls.c +++ b/deps/jni/windows/win32/bridge/AccessBridgeCalls.c @@ -569,9 +569,9 @@ extern "C" { return FALSE; } - AccessibleContext GetAccessibleChildFromContext(long vmID, AccessibleContext ac, jint index) { + AccessibleContext GetAccessibleChildFromContext(long vmID, AccessibleContext ac, jint deps_index) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.GetAccessibleChildFromContext(vmID, ac, index); + return theAccessBridge.GetAccessibleChildFromContext(vmID, ac, deps_index); } return (AccessibleContext) 0; } @@ -711,9 +711,9 @@ extern "C" { * return the row number for a cell at a given index */ jint - getAccessibleTableRow(long vmID, AccessibleTable table, jint index) { + getAccessibleTableRow(long vmID, AccessibleTable table, jint deps_index) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.getAccessibleTableRow(vmID, table, index); + return theAccessBridge.getAccessibleTableRow(vmID, table, deps_index); } return -1; } @@ -722,9 +722,9 @@ extern "C" { * return the column number for a cell at a given index */ jint - getAccessibleTableColumn(long vmID, AccessibleTable table, jint index) { + getAccessibleTableColumn(long vmID, AccessibleTable table, jint deps_index) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.getAccessibleTableColumn(vmID, table, index); + return theAccessBridge.getAccessibleTableColumn(vmID, table, deps_index); } return -1; } @@ -753,9 +753,9 @@ extern "C" { return FALSE; } - BOOL GetAccessibleTextItems(long vmID, AccessibleText at, AccessibleTextItemsInfo *textItems, jint index) { + BOOL GetAccessibleTextItems(long vmID, AccessibleText at, AccessibleTextItemsInfo *textItems, jint deps_index) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.GetAccessibleTextItems(vmID, at, textItems, index); + return theAccessBridge.GetAccessibleTextItems(vmID, at, textItems, deps_index); } return FALSE; } @@ -767,23 +767,23 @@ extern "C" { return FALSE; } - BOOL GetAccessibleTextAttributes(long vmID, AccessibleText at, jint index, AccessibleTextAttributesInfo *attributes) { + BOOL GetAccessibleTextAttributes(long vmID, AccessibleText at, jint deps_index, AccessibleTextAttributesInfo *attributes) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.GetAccessibleTextAttributes(vmID, at, index, attributes); + return theAccessBridge.GetAccessibleTextAttributes(vmID, at, deps_index, attributes); } return FALSE; } - BOOL GetAccessibleTextRect(long vmID, AccessibleText at, AccessibleTextRectInfo *rectInfo, jint index) { + BOOL GetAccessibleTextRect(long vmID, AccessibleText at, AccessibleTextRectInfo *rectInfo, jint deps_index) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.GetAccessibleTextRect(vmID, at, rectInfo, index); + return theAccessBridge.GetAccessibleTextRect(vmID, at, rectInfo, deps_index); } return FALSE; } - BOOL GetAccessibleTextLineBounds(long vmID, AccessibleText at, jint index, jint *startIndex, jint *endIndex) { + BOOL GetAccessibleTextLineBounds(long vmID, AccessibleText at, jint deps_index, jint *startIndex, jint *endIndex) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.GetAccessibleTextLineBounds(vmID, at, index, startIndex, endIndex); + return theAccessBridge.GetAccessibleTextLineBounds(vmID, at, deps_index, startIndex, endIndex); } return FALSE; } @@ -844,7 +844,7 @@ extern "C" { /* * This method is used to iterate through the hyperlinks in a component. It - * returns hypertext information for a component starting at hyperlink index + * returns hypertext information for a component starting at hyperlink deps_index * nStartIndex. No more than MAX_HYPERLINKS AccessibleHypertextInfo objects will * be returned for each call to this method. * returns FALSE on error. @@ -1109,9 +1109,9 @@ extern "C" { /** * Gets the text caret location */ - BOOL getCaretLocation(long vmID, AccessibleContext ac, AccessibleTextRectInfo *rectInfo, jint index) { + BOOL getCaretLocation(long vmID, AccessibleContext ac, AccessibleTextRectInfo *rectInfo, jint deps_index) { if (theAccessBridgeInitializedFlag == TRUE) { - return theAccessBridge.getCaretLocation(vmID, ac, rectInfo, index); + return theAccessBridge.getCaretLocation(vmID, ac, rectInfo, deps_index); } return FALSE; } diff --git a/deps/lua/src/loadlib.c b/deps/lua/src/loadlib.c index 19edaca0fd25a2f418152d58358b2eccb850910a..e754e54b830224b9894a130e72bce59389f87429 100644 --- a/deps/lua/src/loadlib.c +++ b/deps/lua/src/loadlib.c @@ -114,13 +114,13 @@ static void setprogdir (lua_State *L) { static void pusherror (lua_State *L) { - int error = GetLastError(); + int lua_error = GetLastError(); char buffer[128]; if (FormatMessage(FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, error, 0, buffer, sizeof(buffer), NULL)) + NULL, lua_error, 0, buffer, sizeof(buffer), NULL)) lua_pushstring(L, buffer); else - lua_pushfstring(L, "system error %d\n", error); + lua_pushfstring(L, "system error %d\n", lua_error); } static void ll_unloadlib (void *lib) { diff --git a/deps/lz4/inc/lz4.h b/deps/lz4/inc/lz4.h index 43ccb22c9cdb7006b7dab515613580ae4fb8b7a4..7ab1e483a9f53798f6160bd8aaaabd3852e2c146 100644 --- a/deps/lz4/inc/lz4.h +++ b/deps/lz4/inc/lz4.h @@ -1,7 +1,7 @@ /* * LZ4 - Fast LZ compression algorithm * Header File - * Copyright (C) 2011-2017, Yann Collet. + * Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -46,24 +46,31 @@ extern "C" { /** Introduction - LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. Compression can be done in: - a single step (described as Simple Functions) - a single step, reusing a context (described in Advanced Functions) - unbounded multiple steps (described as Streaming compression) - lz4.h provides block compression functions. It gives full buffer control to user. - Decompressing an lz4-compressed block also requires metadata (such as compressed size). - Each application is free to encode such metadata in whichever way it wants. + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. - An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), - take care of encoding standard metadata alongside LZ4-compressed blocks. - If your application requires interoperability, it's recommended to use it. - A library is provided to take care of it, see lz4frame.h. + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. */ /*^*************************************************************** @@ -72,27 +79,28 @@ extern "C" { /* * LZ4_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL -* LZ4LIB_API : +* LZ4LIB_VISIBILITY : * Control library symbols visibility. */ - -#include - +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#elif defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4LIB_API __attribute__ ((__visibility__ ("default"))) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define LZ4LIB_API +# define LZ4LIB_API LZ4LIB_VISIBILITY #endif - /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -101,8 +109,8 @@ extern "C" { #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) -LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */ -LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */ +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */ /*-************************************ @@ -111,42 +119,49 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; /*! * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect + * Increasing memory usage improves compression ratio. + * Reduced memory usage may improve speed, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE # define LZ4_MEMORY_USAGE 14 #endif + /*-************************************ * Simple Functions **************************************/ /*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. -*/ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + * compressedSize : is the exact complete size of the compressed block. + * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); /*-************************************ @@ -155,322 +170,603 @@ LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compress #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) -/*! -LZ4_compressBound() : +/*! LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) inputSize : max supported value is LZ4_MAX_INPUT_SIZE return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) + or 0, if input size is incorrect (too large or negative) */ LZ4LIB_API int LZ4_compressBound(int inputSize); -/*! -LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). */ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/*! -LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails -*/ -LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); - +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'targetDestSize'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + * or 0 if compression fails. + * + * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+): + * the produced compressed content could, in specific circumstances, + * require to be decompressed into a destination buffer larger + * by at least 1 byte than the content to decompress. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ +LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); -/*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); -/*! -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) + * If source stream is detected malformed, function returns a negative result. + * + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity + * + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); /*-********************************************* * Streaming Compression Functions ***********************************************/ -typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); -/*! LZ4_resetStream() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to start compressing a new stream. +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); /*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream_t. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 accept any input as dictionary, + * results are generally better when using Zstandard's Dictionary Builder. * Loading a size of 0 is allowed, and is the same as reset. - * @return : dictionary size, in bytes (necessarily <= 64 KB) + * @return : loaded dictionary size, in bytes (necessarily <= 64 KB) */ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /*! LZ4_compress_fast_continue() : - * Compress content into 'src' using data from previously compressed blocks, improving compression ratio. - * 'dst' buffer must be already allocated. + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * - * Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! - * Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. - * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. - * * @return : size of compressed block - * or 0 if there is an error (typically, compressed data cannot fit into 'dst') - * After an error, the stream status is invalid, it can only be reset or freed. + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its current memory location, + * If last 64KB data cannot be guaranteed to remain available at its current memory location, * save it into a safer place (char* safeBuffer). - * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. - * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. */ -LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); /*-********************************************** * Streaming Decompression Functions * Bufferless synchronous API ************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : - * creation / destruction of streaming decompression tracking structure. - * A tracking structure can be re-used multiple times sequentially. */ + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); /*! LZ4_setStreamDecode() : - * An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. * Use this function to start decompression of a new stream of blocks. - * A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. * @return : 1 if OK, 0 if error */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + /*! LZ4_decompress_*_continue() : * These decoding functions allow decompression of consecutive blocks in "streaming" mode. * A block is an unsplittable entity, it must be presented entirely to a decompression function. - * Decompression functions only accept one block at a time. - * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). - * - * Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : - * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - * maxBlockSize is implementation dependent. It's the maximum size of any single block. + * Decompression functions only accepts one block at a time. + * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. * In which case, encoding and decoding buffers do not need to be synchronized, * and encoding ring buffer can have any size, including small ones ( < 64 KB). - * - _At least_ 64 KB + 8 bytes + maxBlockSize. - * In which case, encoding and decoding buffers do not need to be synchronized, - * and encoding ring buffer can have any size, including larger than decoding buffer. - * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - * and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); +LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_*_usingDict() : * These decoding functions work the same as * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); +LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ -/*^********************************************** +/*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Using these definitions will expose code to API and/or ABI break in future versions of the library. - **************************************/ + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +#define LZ4LIB_STATIC_API LZ4LIB_API +#else +#define LZ4LIB_STATIC_API +#endif + + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_attach_dictionary() : + * This is an experimental API that allows + * efficient use of a static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDict() should + * be expected to work. + * + * Alternatively, the provided dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the first compression call on the stream. + */ +LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); + + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly contrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * Private Definitions + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. + **************************************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ - uint32_t dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - +# include + typedef int8_t LZ4_i8; + typedef uint8_t LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; #else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif -typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ - unsigned int dictSize; -} LZ4_stream_t_internal; +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + const LZ4_byte* dictionary; + const LZ4_stream_t_internal* dictCtx; + LZ4_u32 dictSize; +}; typedef struct { - const unsigned char* externalDict; + const LZ4_byte* externalDict; size_t extDictSize; - const unsigned char* prefixEnd; + const LZ4_byte* prefixEnd; size_t prefixSize; } LZ4_streamDecode_t_internal; -#endif -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * it may change in a future version ! +/*! LZ4_stream_t : + * Do not use below internal definitions directly ! + * Declare or allocate an LZ4_stream_t instead. + * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. + * The structure definition can be convenient for static allocation + * (on stack, or as part of larger structure). + * Init this structure with LZ4_initStream() before first use. + * note : only use this definition in association with static linking ! + * this definition is not API/ABI safe, and may change in future versions. */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(uint64_t)) +#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */ +#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*)) union LZ4_stream_u { - uint64_t table[LZ4_STREAMSIZE_U64]; + void* table[LZ4_STREAMSIZE_VOIDP]; LZ4_stream_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_stream_t */ +}; /* previously typedef'd to LZ4_stream_t */ -/*! - * LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode (or memset()) before first use - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead + */ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size); + + +/*! LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode() before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(uint64_t)) +#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ ) +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { - uint64_t table[LZ4_STREAMDECODESIZE_U64]; + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; LZ4_streamDecode_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_streamDecode_t */ + /*-************************************ * Obsolete Functions **************************************/ /*! Deprecation warnings - Should deprecation warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */ -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ # endif #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ -/* Obsolete compression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +/*! Obsolete compression functions (since v1.7.3) */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); -/* Obsolete decompression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); +/*! Obsolete decompression functions (since v1.8.0) */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); + +/*! Obsolete streaming decoding functions (since v1.7.0) */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : + * These functions used to be faster than LZ4_decompress_safe(), + * but this is no longer the case. They are now slower. + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); -/* Obsolete streaming functions; use new streaming interface whenever possible */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); -/* Obsolete streaming decoding functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); -#endif /* LZ4_H_2983827168210 */ +#endif /* LZ4_H_98237428734687 */ #if defined (__cplusplus) diff --git a/deps/lz4/src/lz4.c b/deps/lz4/src/lz4.c index c48baa63fc1fd7f666e5e8f183e682e262bb0448..9f5e9bfa0839f8e1347d2abb3d867b21ff740215 100644 --- a/deps/lz4/src/lz4.c +++ b/deps/lz4/src/lz4.c @@ -1,6 +1,6 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2017, Yann Collet. + Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -32,7 +32,6 @@ - LZ4 source repository : https://github.com/lz4/lz4 */ - /*-************************************ * Tuning parameters **************************************/ @@ -46,10 +45,16 @@ #endif /* - * ACCELERATION_DEFAULT : + * LZ4_ACCELERATION_DEFAULT : * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 */ -#define ACCELERATION_DEFAULT 1 +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 /*-************************************ @@ -69,9 +74,11 @@ * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define LZ4_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) # define LZ4_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -80,14 +87,33 @@ * LZ4_FORCE_SW_BITCOUNT * Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ # define LZ4_FORCE_SW_BITCOUNT #endif + /*-************************************ * Dependency **************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY +#define LZ4_STATIC_LINKING_ONLY +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* see also "memory routines" below */ @@ -95,10 +121,9 @@ /*-************************************ * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif /* _MSC_VER */ #ifndef LZ4_FORCE_INLINE @@ -117,29 +142,135 @@ # endif /* _MSC_VER */ #endif /* LZ4_FORCE_INLINE */ +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) +#else +# define LZ4_FORCE_O2 +#endif + #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) #endif +#ifndef likely #define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely #define unlikely(expr) expect((expr) != 0, 0) +#endif + +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif /*-************************************ * Memory routines **************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free +#ifdef LZ4_USER_MEMORY_FUNCTIONS +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + #include /* memset, memcpy */ -#define MEM_INIT memset +#define MEM_INIT(p,v,s) memset((p),(v),(s)) + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} /*-************************************ -* Basic Types +* Types **************************************/ +#include #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include typedef uint8_t BYTE; @@ -149,11 +280,14 @@ typedef uint64_t U64; typedef uintptr_t uptrval; #else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; - typedef uint64_t U64; + typedef unsigned long long U64; typedef size_t uptrval; /* generally true, except OpenVMS-64 */ #endif @@ -163,9 +297,31 @@ typedef size_t reg_t; /* 32-bits in x32 mode */ #endif +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + /*-************************************ * Reading and writing into memory **************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if defined(__GNUC__) && (__GNUC__ >= 4) +#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +#else +#define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +#endif + static unsigned LZ4_isLittleEndian(void) { const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ @@ -196,31 +352,31 @@ static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArc static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -#else /* safe and portable access through memcpy() */ +#else /* safe and portable access using memcpy() */ static U16 LZ4_read16(const void* memPtr) { - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static U32 LZ4_read32(const void* memPtr) { - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static reg_t LZ4_read_ARCH(const void* memPtr) { - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static void LZ4_write16(void* memPtr, U16 value) { - memcpy(memPtr, &value, sizeof(value)); + LZ4_memcpy(memPtr, &value, sizeof(value)); } static void LZ4_write32(void* memPtr, U32 value) { - memcpy(memPtr, &value, sizeof(value)); + LZ4_memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ @@ -247,130 +403,216 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } } -static void LZ4_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +LZ4_FORCE_INLINE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; BYTE* const e = (BYTE*)dstEnd; - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_INLINE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; -/*-************************************ -* Error detection -**************************************/ -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d=2) -# include -# define DEBUGLOG(l, ...) { \ - if (l<=LZ4_DEBUG) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ +/* LZ4_memcpy_using_offset() presumes : + * - dstEnd >= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ +LZ4_FORCE_INLINE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch(offset) { + case 1: + MEM_INIT(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); + LZ4_memcpy(&v[4], v, 4); + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} #endif /*-************************************ * Common functions **************************************/ -static unsigned LZ4_NbCommonBytes (register reg_t val) +static unsigned LZ4_NbCommonBytes (reg_t val) { + assert(val != 0); if (LZ4_isLittleEndian()) { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && defined(_M_AMD64) && !defined(LZ4_FORCE_SW_BITCOUNT) + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(int64_t)val) * 0x0218A392CDABBD3FULL)) >> 58]; + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); # endif } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; # endif } } else /* Big Endian CPU */ { if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; # else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } r += (!val); return r; +#endif # endif } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r = 0; - _BitScanReverse( &r, (uint64_t)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; # else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; # endif } } } + #define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { const BYTE* const pStart = pIn; - while (likely(pIn compression ru /*-************************************ * Local Structures and types **************************************/ -typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Like usingExtDict, but everything concerning the preceding + * content is in a separate context, pointed to by + * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table + * entries in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - /*-************************************ * Local Utils @@ -411,13 +672,30 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } +int LZ4_sizeofState(void) { return LZ4_STREAMSIZE; } +/*-************************************ +* Internal Definitions used in Tests +**************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); + +#if defined (__cplusplus) +} +#endif + /*-****************************** * Compression functions ********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -425,15 +703,16 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else + } else { + const U64 prime8bytes = 11400714785074694791ULL; return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } } LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) @@ -442,10 +721,37 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType, + const BYTE* srcBase) { switch (tableType) { + case clearedTable: { /* illegal! */ assert(0); return; } case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } @@ -458,71 +764,161 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ + if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster + * than compressing without a gap. However, compressing with + * currentOffset == 0 is faster still, so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} /** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ -LZ4_FORCE_INLINE int LZ4_compress_generic( + * inlined, to ensure branches are decided at compilation time. + * Presumed already validated at this stage: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( LZ4_stream_t_internal* const cctx, const char* const source, char* const dest, const int inputSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ const int maxOutputSize, - const limitedOutput_directive outputLimited, + const limitedOutput_directive outputDirective, const tableType_t tableType, - const dict_directive dict, + const dict_directive dictDirective, const dictIssue_directive dictIssue, - const U32 acceleration) + const int acceleration) { + int result; const BYTE* ip = (const BYTE*) source; - const BYTE* base; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; const BYTE* const matchlimit = iend - LASTLITERALS; + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = !dictionary ? NULL : (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; + U32 offset = 0; U32 forwardH; - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); + /* If init conditions are not met, we don't have to mark stream + * as having dirty context, since no action was taken yet */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ + if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ + assert(acceleration >= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; } - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizecurrentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizehashTable, tableType, base); @@ -530,50 +926,112 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - ptrdiff_t refDelta = 0; const BYTE* match; BYTE* token; + const BYTE* filledIp; /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; ip = forwardIp; forwardIp += step; step = (searchMatchNb++ >> LZ4_skipTrigger); - if (unlikely(forwardIp > mflimit)) goto _last_literals; + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ lowLimit = dictionary; } else { - refDelta = 0; + match = base + matchIndex; lowLimit = (const BYTE*)source; - } } + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } while(1); } /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + filledIp = ip; + while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; + int len = (int)(litLength - RUN_MASK); *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; *op++ = (BYTE)len; @@ -581,37 +1039,87 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } /* Encode MatchLength */ { unsigned matchCode; - if ((dict==usingExtDict) && (lowLimit==dictionary)) { - const BYTE* limit; - match += refDelta; - limit = ip + (dictEnd-match); + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; + ip += (size_t)matchCode + MINMATCH; if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); matchCode += more; ip += more; } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); } else { matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } if (matchCode >= ML_MASK) { *token += ML_MASK; matchCode -= ML_MASK; @@ -626,41 +1134,89 @@ _next_match: } else *token += (BYTE)(matchCode); } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); anchor = ip; /* Test end of chunk */ - if (ip > mflimit) break; + if (ip >= mflimitPlusOne) break; /* Fill table */ LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: /* Encode Last Literals */ - { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; + { size_t lastRun = (size_t)(iend - anchor); + if ( (outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; @@ -669,252 +1225,182 @@ _last_literals: } else { *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); + return result; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); } int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; - LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } } } - -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { -#if (LZ4_HEAPMODE) - void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctx; - void* const ctxPtr = &ctx; -#endif - - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); - -#if (LZ4_HEAPMODE) - FREEMEM(ctxPtr); -#endif - return result; -} - - -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); -} - - -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} - - -/*-****************************** -* *_destSize() variant -********************************/ - -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } - else *token = (BYTE)(litLength< LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } - -_next_match: - /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; - - /* Encode MatchLength */ - { size_t matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - if (op + ((matchLength+240)/255) > oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } - else *token += (BYTE)(matchLength); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } + } +} - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; +#else + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; +#endif + result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1); + } } } int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (LZ4_HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; #else LZ4_stream_t ctxBody; LZ4_stream_t* ctx = &ctxBody; @@ -936,20 +1422,50 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); + DEBUGLOG(4, "LZ4_createStream %p", lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); return lz4s; } +static size_t LZ4_stream_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ +#endif +} + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); } int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); FREEMEM(LZ4_stream); return (0); } @@ -959,43 +1475,82 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; return 0; } if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; + base = dictEnd - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; + dict->tableType = (U32)tableType; while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); + LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; } - return dict->dictSize; + return (int)dict->dictSize; +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { + const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; } -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; + DEBUGLOG(4, "LZ4_renormDictT"); for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; @@ -1007,16 +1562,29 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) } -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) { + const tableType_t tableType = byU32; LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); + + LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ + && (dictEnd != (const BYTE*)source) ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = (const BYTE*)source; + } /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; @@ -1030,46 +1598,61 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* prefix mode : source data follows dictionary */ if (dictEnd == (const BYTE*)source) { - int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); } /* external dictionary mode */ { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; return result; } } -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) { LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); + LZ4_renormDictT(streamPtr, srcSize); - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; + streamPtr->dictSize = (U32)srcSize; return result; } @@ -1087,10 +1670,12 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; @@ -1100,212 +1685,602 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -/*-***************************** -* Decompression functions -*******************************/ +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +/* Read the variable-length literal or match length. + * + * ip - pointer to use as input. + * lencheck - end ip. Return an error if ip advances >= lencheck. + * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. + * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. + * error (output) - error code. Should be set to 0 before call. + */ +typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error; +LZ4_FORCE_INLINE unsigned +read_variable_length(const BYTE**ip, const BYTE* lencheck, + int loop_check, int initial_check, + variable_length_error* error) +{ + U32 length = 0; + U32 s; + if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = initial_error; + return length; + } + do { + s = **ip; + (*ip)++; + length += s; + if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = loop_error; + return length; + } + } while (s==255); + + return length; +} + /*! LZ4_decompress_generic() : * This generic decompression function covers all use cases. * It shall be instantiated several times, using different sets of directives. * Note that it is important for performance that this function really get inlined, * in order to remove useless branches during compilation optimization. */ -LZ4_FORCE_INLINE int LZ4_decompress_generic( +LZ4_FORCE_INLINE int +LZ4_decompress_generic( const char* const src, char* const dst, int srcSize, int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dst when no prefix */ + endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */ + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ const BYTE* const dictStart, /* only if dict==usingExtDict */ const size_t dictSize /* note : = 0 if noDict */ ) { - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; + if (src == NULL) { return -1; } - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - /* Main Loop : decode sequences */ - while (1) { - size_t length; + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + const BYTE* match; size_t offset; + unsigned token; + size_t length; - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + if (endOnInput) { assert(ip < iend); } + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if (endOnInput) { /* LZ4_decompress_safe() */ + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, cpy); + } else { /* LZ4_decompress_fast() */ + if (cpy>oend-8) { goto safe_literal_copy; } + LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and only relies on end-of-block properties */ + } + ip += length; op = cpy; } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + cpy = op+length; + if (endOnInput) { /* LZ4_decompress_safe() */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* We don't need to check oend, since we check it once for each loop below */ + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } + /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ + LZ4_memcpy(op, ip, 16); + } else { /* LZ4_decompress_fast() */ + /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and relies on end-of-block properties */ + LZ4_memcpy(op, ip, 8); + if (length > 8) { LZ4_memcpy(op+8, ip+8, 8); } + } + ip += length; op = cpy; } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + assert(match <= op); + + /* get matchlength */ + length = token & ML_MASK; + + if (length == ML_MASK) { + variable_length_error error = ok; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); + if (error != ok) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + + /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); + } else { + goto _output_error; /* end-of-block condition violated */ + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ } - length += MINMATCH; + safe_decode: +#endif - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + while (1) { + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + /* Copy the literals */ + LZ4_memcpy(op, ip, endOnInput ? 16 : 8); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } + + /* copy literals */ + cpy = op+length; +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + assert(endOnInput); + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + } else { + /* We must be on the last sequence because of the parsing limitations so check + * that we exactly regenerate the original size (must be exact when !endOnInput). + */ + if ((!endOnInput) && (cpy != oend)) { goto _output_error; } + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { + DEBUGLOG(6, "should have been last run of literals") + DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); + DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend); + goto _output_error; + } + } + memmove(op, ip, length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */ + ip += length; op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { + break; + } } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + + _copy_match: + if (length == ML_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); + if (error != ok) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { break; } + continue; + } - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; - - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; + if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op+4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ } - op=cpy; /* correction */ - } - - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-src))-1; + /* end of decoding */ + if (endOnInput) { + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + } else { + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + } + + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } } +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + endOnInputSize, decode_full_block, noDict, + (BYTE*)dest, NULL, 0); } -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + endOnInputSize, partial_decode, + noDict, (BYTE*)dst, NULL, 0); } +LZ4_FORCE_O2 int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); } +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + /* LZ4_decompress_fast doesn't validate match offsets, + * and thus serves well with any prefixed dictionary. */ + return LZ4_decompress_fast(source, dest, originalSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_INLINE +int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} /*===== streaming decompression functions =====*/ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); + LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */ return lz4s; } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - if (!LZ4_stream) return 0; /* support free on NULL */ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } -/*! - * LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { @@ -1317,6 +2292,25 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti return 1; } +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. @@ -1324,52 +2318,75 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ +LZ4_FORCE_O2 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (result <= 0) return result; - lz4sd->prefixSize += result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; lz4sd->prefixEnd += result; } else { + /* The buffer wraps around, or they're switching to another buffer. */ lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = result; + lz4sd->prefixSize = (size_t)result; lz4sd->prefixEnd = (BYTE*)dest + result; } return result; } +LZ4_FORCE_O2 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; + assert(originalSize >= 0); - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (lz4sd->prefixSize == 0) { + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); if (result <= 0) return result; - lz4sd->prefixSize += originalSize; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) + result = LZ4_decompress_fast(source, dest, originalSize); + else + result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; lz4sd->prefixEnd += originalSize; } else { lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = originalSize; + lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } @@ -1384,32 +2401,27 @@ Advanced decoding functions : the dictionary must be explicitly provided within parameters */ -LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_fast(source, dest, originalSize); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } @@ -1417,64 +2429,67 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres * Obsolete Functions ***************************************************/ /* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* src, char* dest, int srcSize) +{ + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} /* -These function names are deprecated and should no longer be used. +These decompression functions are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. - LZ4_uncompress is totally equivalent to LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe */ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} /* Obsolete Streaming functions */ -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) -{ - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; -} +int LZ4_sizeofStreamState(void) { return LZ4_STREAMSIZE; } int LZ4_resetStreamState(void* state, char* inputBuffer) { - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); return 0; } void* LZ4_create (char* inputBuffer) { - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; + (void)inputBuffer; + return LZ4_createStream(); } -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +char* LZ4_slideInputBuffer (void* state) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; } #endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/deps/regex/regcomp.c b/deps/regex/regcomp.c index 114b299c61275b5a3709ebec9aff17cbc5665361..5d5794928989a0edc1124e62facf2a72a7704c12 100644 --- a/deps/regex/regcomp.c +++ b/deps/regex/regcomp.c @@ -120,7 +120,7 @@ static reg_errcode_t build_charclass (re_bitset_ptr_t sbcset, static bin_tree_t *build_word_op (re_dfa_t *dfa, int not, reg_errcode_t *err); static void free_bin_tree (bin_tree_t *tree); static bin_tree_t *create_tree (bin_tree_t *left, bin_tree_t *right, - re_token_type_t type, int index); + re_token_type_t type, int deps_index); static bin_tree_t *duplicate_tree (const bin_tree_t *src, re_dfa_t *dfa); /* This table gives an error message for each of the error codes listed @@ -3447,11 +3447,11 @@ free_charset (re_charset_t *cset) Note: This function automatically free left and right if malloc fails. */ static bin_tree_t * -create_tree (left, right, type, index) +create_tree (left, right, type, deps_index) bin_tree_t *left; bin_tree_t *right; re_token_type_t type; - int index; + int deps_index; { bin_tree_t *tree; tree = re_malloc (bin_tree_t, 1); @@ -3465,7 +3465,7 @@ create_tree (left, right, type, index) tree->left = left; tree->right = right; tree->type = type; - tree->node_idx = index; + tree->node_idx = deps_index; tree->first = -1; tree->next = -1; re_node_set_init_empty (&tree->eclosure); diff --git a/documentation20/cn/09.connections/docs.md b/documentation20/cn/09.connections/docs.md index b47f297ae0a68c91e5d38aad000acdb14591283d..d341b31b01badfe0db97d6267243e079ccdda0eb 100644 --- a/documentation20/cn/09.connections/docs.md +++ b/documentation20/cn/09.connections/docs.md @@ -3,7 +3,7 @@ ## Grafana -TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/)快速集成搭建数据监测报警系统，整个过程无需任何代码开发，TDengine中数据表中内容可以在仪表盘(DashBoard)上进行可视化展现。 +TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/)快速集成搭建数据监测报警系统，整个过程无需任何代码开发，TDengine 中数据表中内容可以在仪表盘(DashBoard)上进行可视化展现。关于TDengine插件的使用您可以在[GitHub](https://github.com/taosdata/grafanaplugin/blob/master/README.md)中了解更多。 ### 安装Grafana @@ -11,12 +11,24 @@ TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/) ### 配置Grafana -TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin目录下。 +TDengine 的 Grafana 插件请从 下载。 -以CentOS 7.2操作系统为例，将grafanaplugin目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。 +```bash +GF_VERSION=3.1.1 +wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip +``` + +以 CentOS 7.2 操作系统为例，将插件包解压到 /var/lib/grafana/plugins 目录下，重新启动 grafana 即可。 ```bash -sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tdengine +sudo unzip tdengine-datasource-$GF_VERSION.zip -d /var/lib/grafana/plugins/ +``` + +Grafana 7.3+ / 8.x 版本会对插件进行签名检查，因此还需要在 grafana.ini 文件中增加如下行，才能正确使用插件： + +```ini +[plugins] +allow_loading_unsigned_plugins = tdengine-datasource ``` ### 使用 Grafana @@ -55,7 +67,6 @@ sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tde * ALIAS BY：可设置当前查询别名。 * GENERATE SQL： 点击该按钮会自动替换相应变量，并生成最终执行的语句。 - 按照默认提示查询当前 TDengine 部署所在服务器指定间隔系统内存平均使用量如下： ![img](page://images/connections/create_dashboard2.jpg) @@ -64,16 +75,15 @@ sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tde #### 导入 Dashboard -在 Grafana 插件目录 /usr/local/taos/connector/grafana/tdengine/dashboard/ 下提供了一个 `tdengine-grafana.json` 可导入的 dashboard。 +我们提供一个 TDengine Dashboard 可以作为 TDengine 集群的监控可视化工具使用，见 [Grafana Dashboard 15146](https://grafana.com/grafana/dashboards/15146)。 -点击左侧 `Import` 按钮，并上传 `tdengine-grafana.json` 文件： +点击左侧 `Import` 按钮，选择 **Grafana.com Dashboard**,j将id `15146` 填入并加载： ![img](page://images/connections/import_dashboard1.jpg) 导入完成之后可看到如下效果： -![img](page://images/connections/import_dashboard2.jpg) - +![img](../images/connections/dashboard-15146.png) ## MATLAB diff --git a/documentation20/cn/14.devops/01.telegraf/docs.md b/documentation20/cn/14.devops/01.telegraf/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..04765602dab18fbacf7d92d44ca324db660c0ac4 --- /dev/null +++ b/documentation20/cn/14.devops/01.telegraf/docs.md @@ -0,0 +1,73 @@ +# 使用 TDengine + Telegraf + Grafana 快速搭建 IT 运维展示系统 + +## 背景介绍 +TDengine是涛思数据专为物联网、车联网、工业互联网、IT运维等设计和优化的大数据平台。自从 2019年 7 月开源以来，凭借创新的数据建模设计、快捷的安装方式、易用的编程接口和强大的数据写入查询性能博得了大量时序数据开发者的青睐。 + +IT 运维监测数据通常都是对时间特性比较敏感的数据，例如： +- 系统资源指标：CPU、内存、IO、带宽等。 +- 软件系统指标：存活状态、连接数目、请求数目、超时数目、错误数目、响应时间、服务类型及其他与业务有关的指标。 + +当前主流的 IT 运维系统通常包含一个数据采集模块，一个数据存储模块，和一个可视化显示模块。Telegraf 和 Grafana 分别是当前最流行的数据采集模块和可视化显示模块之一。而数据存储模块可供选择的软件比较多，其中 OpenTSDB 或 InfluxDB 比较流行。而 TDengine 作为新兴的时序大数据平台，具备极强的高性能、高可靠、易管理、易维护的优势。 + +本文介绍不需要写一行代码，通过简单修改几行配置文件，就可以快速搭建一个基于 TDengine + Telegraf + Grafana 的 IT 运维系统。架构如下图： + +![IT-DevOps-Solutions-Telegraf.png](../../images/IT-DevOps-Solutions-Telegraf.png) + + +## 安装步骤 + +### 安装 Telegraf，Grafana 和 TDengine +安装 Telegraf、Grafana 和 TDengine 请参考相关官方文档。 + +### Telegraf +请参考[官方文档](https://portal.influxdata.com/downloads/)。 + +### Grafana +请参考[官方文档](https://grafana.com/grafana/download)。 + +### 安装 TDengine +从涛思数据官网[下载](http://taosdata.com/cn/all-downloads/）页面下载最新 TDengine-server 2.3.0.0 或以上版本安装。 + + +## 数据链路设置 +### 下载 TDengine 插件到 grafana 插件目录 + +```bash +1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.1/tdengine-datasource-3.1.1.zip +2. sudo unzip tdengine-datasource-3.1.1.zip -d /var/lib/grafana/plugins/ +3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine +4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini +5. sudo systemctl restart grafana-server.service +``` + +### 修改 /etc/telegraf/telegraf.conf +配置方法，在 /etc/telegraf/telegraf.conf 增加如下文字，其中 database name 请填写希望在 TDengine 保存 Telegraf 数据的数据库名，TDengine server/cluster host、username和 password 填写 TDengine 实际值： +``` +[[outputs.http]] + url = "http://:6041/influxdb/v1/write?db=" + method = "POST" + timeout = "5s" + username = "" + password = "" + data_format = "influx" + influx_max_line_bytes = 250 +``` + +然后重启 telegraf： +``` +sudo systemctl start telegraf +``` + + +### 导入 Dashboard + +使用 Web 浏览器访问 IP:3000 登录 Grafana 界面，系统初始用户名密码为 admin/admin。 +点击左侧齿轮图标并选择 Plugins，应该可以找到 TDengine data source 插件图标。 +点击左侧加号图标并选择 Import，从 https://github.com/taosdata/grafanaplugin/blob/master/examples/telegraf/grafana/dashboards/telegraf-dashboard-v0.1.0.json 下载 dashboard JSON 文件后导入。之后可以看到如下界面的仪表盘： + +![IT-DevOps-Solutions-telegraf-dashboard.png](../../images/IT-DevOps-Solutions-telegraf-dashboard.png) + + +## 总结 + +以上演示如何快速搭建一个完整的 IT 运维展示系统。得力于 TDengine 2.3.0.0 版本中新增的 schemaless 协议解析功能，以及强大的生态软件适配能力，用户可以短短数分钟就可以搭建一个高效易用的 IT 运维系统。TDengine 强大的数据写入查询性能和其他丰富功能请参考官方文档和产品落地案例。 diff --git a/documentation20/cn/14.devops/02.collectd/docs.md b/documentation20/cn/14.devops/02.collectd/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..a35772bb498d426a1f44a9e7eb0bea61b51f92a5 --- /dev/null +++ b/documentation20/cn/14.devops/02.collectd/docs.md @@ -0,0 +1,79 @@ +# 使用 TDengine + collectd/StatsD + Grafana 快速搭建 IT 运维监控系统 + +## 背景介绍 +TDengine是涛思数据专为物联网、车联网、工业互联网、IT运维等设计和优化的大数据平台。自从 2019年 7 月开源以来，凭借创新的数据建模设计、快捷的安装方式、易用的编程接口和强大的数据写入查询性能博得了大量时序数据开发者的青睐。 + +IT 运维监测数据通常都是对时间特性比较敏感的数据，例如： +- 系统资源指标：CPU、内存、IO、带宽等。 +- 软件系统指标：存活状态、连接数目、请求数目、超时数目、错误数目、响应时间、服务类型及其他与业务有关的指标。 + +当前主流的 IT 运维系统通常包含一个数据采集模块，一个数据存储模块，和一个可视化显示模块。collectd / statsD 作为老牌开源数据采集工具，具有广泛的用户群。但是 collectd / StatsD 自身功能有限，往往需要配合 Telegraf、Grafana 以及时序数据库组合搭建成为完整的监控系统。而 TDengine 新版本支持多种数据协议接入，可以直接接受 collectd 和 statsD 的数据写入，并提供 Grafana dashboard 进行图形化展示。 + +本文介绍不需要写一行代码，通过简单修改几行配置文件，就可以快速搭建一个基于 TDengine + collectd / statsD + Grafana 的 IT 运维系统。架构如下图： + +![IT-DevOps-Solutions-Collectd-StatsD.png](../../images/IT-DevOps-Solutions-Collectd-StatsD.png) + +## 安装步骤 +安装 collectd， StatsD， Grafana 和 TDengine 请参考相关官方文档。 + +### 安装 collectd +请参考[官方文档](https://collectd.org/documentation.shtml)。 + +### 安装 StatsD +请参考[官方文档](https://github.com/statsd/statsd)。 + +### 安装 Grafana +请参考[官方文档](https://grafana.com/grafana/download)。 + +### 安装 TDengine +从涛思数据官网[下载](http://taosdata.com/cn/all-downloads/）页面下载最新 TDengine-server 2.3.0.0 或以上版本安装。 + +## 数据链路设置 +### 复制 TDengine 插件到 grafana 插件目录 + +```bash +1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.1/tdengine-datasource-3.1.1.zip +2. sudo unzip tdengine-datasource-3.1.1.zip -d /var/lib/grafana/plugins/ +3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine +4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini +5. sudo systemctl restart grafana-server.service +``` + +### 配置 collectd +在 /etc/collectd/collectd.conf 文件中增加如下内容，其中 host 和 port 请填写 TDengine 和 BLM3 配置的实际值： +``` +LoadPlugin network + + Server "" "" + + +sudo systemctl start collectd +``` + +### 配置 StatsD +在 config.js 文件中增加如下内容后启动 StatsD，其中 host 和 port 请填写 TDengine 和 BLM3 配置的实际值： +``` +backends 部分添加 "./backends/repeater" +repeater 部分添加 { host:'', port: } +``` + +### 导入 Dashboard + +使用 Web 浏览器访问运行 Grafana 的服务器的3000端口 host:3000 登录 Grafana 界面，系统初始用户名密码为 admin/admin。 +点击左侧齿轮图标并选择 Plugins，应该可以找到 TDengine data source 插件图标。 + +#### 导入 collectd 仪表盘 + +从 https://github.com/taosdata/grafanaplugin/blob/master/examples/collectd/grafana/dashboards/collect-metrics-with-tdengine-v0.1.0.json 下载 dashboard json 文件，点击左侧加号图标并选择 Import，按照界面提示选择 JSON 文件导入。之后可以看到如下界面的仪表盘： + +![IT-DevOps-Solutions-collectd-dashboard.png](../../images/IT-DevOps-Solutions-collectd-dashboard.png) + +#### 导入 StatsD 仪表盘 + +从 https://github.com/taosdata/grafanaplugin/blob/master/examples/statsd/dashboards/statsd-with-tdengine-v0.1.0.json 下载 dashboard json 文件，点击左侧加号图标并选择 Import，按照界面提示导入JSON文件。之后可以看到如下界面的仪表盘： +![IT-DevOps-Solutions-statsd-dashboard.png](../../images/IT-DevOps-Solutions-statsd-dashboard.png) + +## 总结 +TDengine 作为新兴的时序大数据平台，具备极强的高性能、高可靠、易管理、易维护的优势。得力于 TDengine 2.3.0.0 版本中新增的 schemaless 协议解析功能，以及强大的生态软件适配能力，用户可以短短数分钟就可以搭建一个高效易用的 IT 运维系统或者适配一个已存在的系统。 + +TDengine 强大的数据写入查询性能和其他丰富功能请参考官方文档和产品成功落地案例。 diff --git a/documentation20/cn/images/connections/dashboard-15146.png b/documentation20/cn/images/connections/dashboard-15146.png new file mode 100644 index 0000000000000000000000000000000000000000..3eb240ad8ad648953e32f27e674e2a9171ed9af8 Binary files /dev/null and b/documentation20/cn/images/connections/dashboard-15146.png differ diff --git a/documentation20/en/09.connections/docs.md b/documentation20/en/09.connections/docs.md index e759da31677a5344a0f6578c751c4b77f86a43db..6a398dfdf70f67ce4d5a520ec9bc9ebf3c6e993e 100644 --- a/documentation20/en/09.connections/docs.md +++ b/documentation20/en/09.connections/docs.md @@ -12,12 +12,17 @@ https://grafana.com/grafana/download. ### Configure Grafana -TDengine Grafana plugin is in the /usr/local/taos/connector/grafanaplugin directory. +Download grafana plugin from . + +```bash +GF_VERSION=3.1.1 +wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip +``` Taking Centos 7.2 as an example, just copy grafanaplugin directory to /var/lib/grafana/plugins directory and restart Grafana. ```bash -sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tdengine +sudo unzip tdengine-datasource-$GF_VERSION.zip /var/lib/grafana/plugins/ ``` ### Use Grafana @@ -64,15 +69,15 @@ According to the default prompt, query the average system memory usage at the sp #### Import Dashboard -A `tdengine-grafana.json` importable dashboard is provided under the Grafana plug-in directory/usr/local/taos/connector/grafana/tdengine/dashboard/. +We provide an example dashboard [Grafana Dashboard 15146](https://grafana.com/grafana/dashboards/15146)。 -Click the `Import` button on the left panel and upload the `tdengine-grafana.json` file: +Click the `Import` button on the left panel and load the grafana id: ![img](page://images/connections/import_dashboard1.jpg) You can see as follows after Dashboard imported. -![img](page://images/connections/import_dashboard2.jpg) +![img](../images/connections/dashboard-15146.png) ## MATLAB diff --git a/documentation20/en/images/connections/dashboard-15146.png b/documentation20/en/images/connections/dashboard-15146.png new file mode 100644 index 0000000000000000000000000000000000000000..3eb240ad8ad648953e32f27e674e2a9171ed9af8 Binary files /dev/null and b/documentation20/en/images/connections/dashboard-15146.png differ diff --git a/packaging/cfg/kinghistorian.cfg b/packaging/cfg/kinghistorian.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0d0d0b9f3eaae6539f9e431d51bb532270179226 --- /dev/null +++ b/packaging/cfg/kinghistorian.cfg @@ -0,0 +1,286 @@ +######################################################## +# # +# KingHistorian Configuration # +# Any questions, please email support@wellintech.com # +# # +######################################################## + +# first fully qualified domain name (FQDN) for KingHistorian system +# firstEp hostname:6030 + +# local fully qualified domain name (FQDN) +# fqdn hostname + +# first port number for the connection (12 continuous UDP/TCP port number are used) +# serverPort 6030 + +# log file's directory +# logDir /var/log/kinghistorian + +# data file's directory +# dataDir /var/lib/kinghistorian + +# temporary file's directory +# tempDir /tmp/ + +# the arbitrator's fully qualified domain name (FQDN) for KingHistorian system, for cluster only +# arbitrator arbitrator_hostname:6042 + +# number of threads per CPU core +# numOfThreadsPerCore 1.0 + +# number of threads to commit cache data +# numOfCommitThreads 4 + +# the proportion of total CPU cores available for query processing +# 2.0: the query threads will be set to double of the CPU cores. +# 1.0: all CPU cores are available for query processing [default]. +# 0.5: only half of the CPU cores are available for query. +# 0.0: only one core available. +# ratioOfQueryCores 1.0 + +# the last_row/first/last aggregator will not change the original column name in the result fields +keepColumnName 1 + +# number of management nodes in the system +# numOfMnodes 3 + +# enable/disable backuping vnode directory when removing vnode +# vnodeBak 1 + +# enable/disable installation / usage report +# telemetryReporting 1 + +# enable/disable load balancing +# balance 1 + +# role for dnode. 0 - any, 1 - mnode, 2 - dnode +# role 0 + +# max timer control blocks +# maxTmrCtrl 512 + +# time interval of system monitor, seconds +# monitorInterval 30 + +# number of seconds allowed for a dnode to be offline, for cluster only +# offlineThreshold 864000 + +# RPC re-try timer, millisecond +# rpcTimer 300 + +# RPC maximum time for ack, seconds. +# rpcMaxTime 600 + +# time interval of dnode status reporting to mnode, seconds, for cluster only +# statusInterval 1 + +# time interval of heart beat from shell to dnode, seconds +# shellActivityTimer 3 + +# minimum sliding window time, milli-second +# minSlidingTime 10 + +# minimum time window, milli-second +# minIntervalTime 10 + +# maximum delay before launching a stream computation, milli-second +# maxStreamCompDelay 20000 + +# maximum delay before launching a stream computation for the first time, milli-second +# maxFirstStreamCompDelay 10000 + +# retry delay when a stream computation fails, milli-second +# retryStreamCompDelay 10 + +# the delayed time for launching a stream computation, from 0.1(default, 10% of whole computing time window) to 0.9 +# streamCompDelayRatio 0.1 + +# max number of vgroups per db, 0 means configured automatically +# maxVgroupsPerDb 0 + +# max number of tables per vnode +# maxTablesPerVnode 1000000 + +# cache block size (Mbyte) +# cache 16 + +# number of cache blocks per vnode +# blocks 6 + +# number of days per DB file +# days 10 + +# number of days to keep DB file +# keep 3650 + +# minimum rows of records in file block +# minRows 100 + +# maximum rows of records in file block +# maxRows 4096 + +# the number of acknowledgments required for successful data writing +# quorum 1 + +# enable/disable compression +# comp 2 + +# write ahead log (WAL) level, 0: no wal; 1: write wal, but no fysnc; 2: write wal, and call fsync +# walLevel 1 + +# if walLevel is set to 2, the cycle of fsync being executed, if set to 0, fsync is called right away +# fsync 3000 + +# number of replications, for cluster only +# replica 1 + +# the compressed rpc message, option: +# -1 (no compression) +# 0 (all message compressed), +# > 0 (rpc message body which larger than this value will be compressed) +# compressMsgSize -1 + +# max length of an SQL +# maxSQLLength 65480 + +# max length of WildCards +# maxWildCardsLength 100 + +# the maximum number of records allowed for super table time sorting +# maxNumOfOrderedRes 100000 + +# system time zone +# timezone Asia/Shanghai (CST, +0800) +# system time zone (for windows 10) +# timezone UTC-8 + +# system locale +# locale en_US.UTF-8 + +# default system charset +# charset UTF-8 + +# max number of connections allowed in dnode +# maxShellConns 5000 + +# max number of connections allowed in client +# maxConnections 5000 + +# stop writing logs when the disk size of the log folder is less than this value +# minimalLogDirGB 0.1 + +# stop writing temporary files when the disk size of the tmp folder is less than this value +# minimalTmpDirGB 0.1 + +# if disk free space is less than this value, khserver service exit directly within startup process +# minimalDataDirGB 0.1 + +# One mnode is equal to the number of vnode consumed +# mnodeEqualVnodeNum 4 + +# enbale/disable http service +# http 1 + +# enable/disable system monitor +# monitor 1 + +# enable/disable recording the SQL statements via restful interface +# httpEnableRecordSql 0 + +# number of threads used to process http requests +# httpMaxThreads 2 + +# maximum number of rows returned by the restful interface +# restfulRowLimit 10240 + +# The following parameter is used to limit the maximum number of lines in log files. +# max number of lines per log filters +# numOfLogLines 10000000 + +# enable/disable async log +# asyncLog 1 + +# time of keeping log files, days +# logKeepDays 0 + + +# The following parameters are used for debug purpose only. +# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR +# 131: output warning and error +# 135: output debug, warning and error +# 143: output trace, debug, warning and error to log +# 199: output debug, warning and error to both screen and file +# 207: output trace, debug, warning and error to both screen and file + +# debug flag for all log type, take effect when non-zero value +# debugFlag 0 + +# debug flag for meta management messages +# mDebugFlag 135 + +# debug flag for dnode messages +# dDebugFlag 135 + +# debug flag for sync module +# sDebugFlag 135 + +# debug flag for WAL +# wDebugFlag 135 + +# debug flag for SDB +# sdbDebugFlag 135 + +# debug flag for RPC +# rpcDebugFlag 131 + +# debug flag for TIMER +# tmrDebugFlag 131 + +# debug flag for KingHistorian client +# cDebugFlag 131 + +# debug flag for JNI +# jniDebugFlag 131 + +# debug flag for storage +# uDebugFlag 131 + +# debug flag for http server +# httpDebugFlag 131 + +# debug flag for monitor +# monDebugFlag 131 + +# debug flag for query +# qDebugFlag 131 + +# debug flag for vnode +# vDebugFlag 131 + +# debug flag for TSDB +# tsdbDebugFlag 131 + +# debug flag for continue query +# cqDebugFlag 131 + +# enable/disable recording the SQL in kinghistorian client +# enableRecordSql 0 + +# generate core file when service crash +# enableCoreFile 1 + +# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden +# maxBinaryDisplayWidth 30 + +# enable/disable stream (continuous query) +# stream 1 + +# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode +# retrieveBlockingModel 0 + +# the maximum allowed query buffer size in MB during query processing for each data node +# -1 no limit (default) +# 0 no query allowed, queries are disabled +# queryBufferSize -1 + diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index a169bf2ba02d67d7c540d3dc3f017324bbc15fc8..f1a7e9bb9d3b4cfc86f55c68ec2b59417a371ac0 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -58,12 +58,6 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples -if [ -d "${top_dir}/src/connector/grafanaplugin/dist" ]; then - cp -r ${top_dir}/src/connector/grafanaplugin/dist ${pkg_dir}${install_home_path}/connector/grafanaplugin -else - echo "grafanaplugin bundled directory not found!" - exit 1 -fi cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector diff --git a/packaging/deb/nginxd b/packaging/deb/nginxd new file mode 100644 index 0000000000000000000000000000000000000000..bdc50fb0464df504977249182857f06c2ec2ccbf --- /dev/null +++ b/packaging/deb/nginxd @@ -0,0 +1,46 @@ +#!/bin/bash +#Startup script for the nginx Web Server +# chkconfig: 2345 99 01 +# description: Nginx For TDengine Service. +# +# +### BEGIN INIT INFO +# Provides: nginx +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts nginx +# Description: Starts nginx. +### END INIT INFO + +nginx=/usr/local/nginxd/sbin/nginx +case $1 in + start) + echo -n "Starting Nginx" + $nginx + echo " done." + ;; + stop) + echo -n "Stopping Nginx" + $nginx -s stop + echo " done." + ;; + test) + $nginx -t + echo "Success." + ;; + reload) + echo -n "Reloading Nginx" + $nginx -s reload + echo " done." + ;; + restart) + $nginx -s reload + echo "reload done." + ;; + *) + echo "Usage: $0 {start|restart|reload|stop|test|show}" + ;; +esac + diff --git a/packaging/release.sh b/packaging/release.sh index 8ed73e9c85b42c2e42e11657dd30db50c1a4d779..dfa9ae9281445f07a2cdba8a89a0c91a430e0138 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -11,7 +11,7 @@ set -e # -V [stable | beta] # -l [full | lite] # -s [static | dynamic] -# -d [taos | power | tq ] +# -d [taos | power | tq | pro | kh | jh] # -n [2.0.0.3] # -m [2.0.0.0] @@ -22,7 +22,7 @@ cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] -dbName=taos # [taos | power | tq | pro] +dbName=taos # [taos | power | tq | pro | kh | jh] allocator=glibc # [glibc | jemalloc] verNumber="" verNumberComp="1.0.0.0" @@ -78,7 +78,7 @@ do echo " -l [full | lite] " echo " -a [glibc | jemalloc] " echo " -s [static | dynamic] " - echo " -d [taos | power | tq | pro] " + echo " -d [taos | power | tq | pro | kh | jh] " echo " -n [version number] " echo " -m [compatible version number] " exit 0 @@ -192,16 +192,301 @@ else allocator_macro="" fi +# for powerdb +if [[ "$dbName" == "power" ]]; then + # cmake/install.inc + sed -i "s/C:\/TDengine/C:\/PowerDB/g" ${top_dir}/cmake/install.inc + sed -i "s/taos\.exe/power\.exe/g" ${top_dir}/cmake/install.inc + sed -i "s/taosdemo\.exe/powerdemo\.exe/g" ${top_dir}/cmake/install.inc + # src/kit/shell/inc/shell.h + sed -i "s/taos_history/power_history/g" ${top_dir}/src/kit/shell/inc/shell.h + # src/inc/taosdef.h + sed -i "s/\"taosdata\"/\"powerdb\"/g" ${top_dir}/src/inc/taosdef.h + # src/util/src/tconfig.c + sed -i "s/taos config/power config/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/taos\.cfg/power\.cfg/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/etc\/taos/etc\/power/g" ${top_dir}/src/util/src/tconfig.c + # src/kit/taosdemo/taosdemo.c + sed -i "s/\"taosdata\"/\"powerdb\"/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + # src/util/src/tlog.c + sed -i "s/log\/taos/log\/power/g" ${top_dir}/src/util/src/tlog.c + # src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/Power/g" ${top_dir}/src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/Power/g" ${top_dir}/src/dnode/src/dnodeMain.c + sed -i "s/taosdlog/powerdlog/g" ${top_dir}/src/dnode/src/dnodeMain.c + # src/client/src/tscSystem.c + sed -i "s/taoslog/powerlog/g" ${top_dir}/src/client/src/tscSystem.c + # src/util/src/tnote.c + sed -i "s/taosinfo/powerinfo/g" ${top_dir}/src/util/src/tnote.c + # src/dnode/CMakeLists.txt + sed -i "s/taos\.cfg/power\.cfg/g" ${top_dir}/src/dnode/CMakeLists.txt + # src/kit/taosdump/taosdump.c + sed -i "s/TDengine/Power/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/Default is taosdata/Default is power/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/\"taosdata\"/\"powerdb\"/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/taos\/taos\.cfg/power\/power\.cfg/g" ${top_dir}/src/kit/taosdump/taosdump.c + # src/os/src/linux/linuxEnv.c + sed -i "s/etc\/taos/etc\/power/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/lib\/taos/lib\/power/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/log\/taos/log\/power/g" ${top_dir}/src/os/src/linux/linuxEnv.c + # src/os/src/windows/wEnv.c + sed -i "s/TDengine/PowerDB/g" ${top_dir}/src/os/src/windows/wEnv.c + # src/kit/shell/src/shellEngine.c + sed -i "s/TDengine shell/PowerDB shell/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/2020 by TAOS Data, Inc/2020 by PowerDB, Inc/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\"taos> \"/\"power> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\" -> \"/\" -> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/prompt_size = 6/prompt_size = 7/g" ${top_dir}/src/kit/shell/src/shellEngine.c +fi + +# for tq +if [[ "$dbName" == "tq" ]]; then + # cmake/install.inc + sed -i "s/C:\/TDengine/C:\/TQueue/g" ${top_dir}/cmake/install.inc + sed -i "s/taos\.exe/tq\.exe/g" ${top_dir}/cmake/install.inc + sed -i "s/taosdemo\.exe/tqdemo\.exe/g" ${top_dir}/cmake/install.inc + # src/kit/shell/inc/shell.h + sed -i "s/taos_history/tq_history/g" ${top_dir}/src/kit/shell/inc/shell.h + # src/inc/taosdef.h + sed -i "s/\"taosdata\"/\"tqueue\"/g" ${top_dir}/src/inc/taosdef.h + # src/util/src/tconfig.c + sed -i "s/taos config/tq config/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/taos\.cfg/tq\.cfg/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/etc\/taos/etc\/tq/g" ${top_dir}/src/util/src/tconfig.c + # src/kit/taosdemo/taosdemo.c + sed -i "s/\"taosdata\"/\"tqueue\"/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + # src/util/src/tlog.c + sed -i "s/log\/taos/log\/tq/g" ${top_dir}/src/util/src/tlog.c + # src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/TQueue/g" ${top_dir}/src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/TQueue/g" ${top_dir}/src/dnode/src/dnodeMain.c + sed -i "s/taosdlog/tqdlog/g" ${top_dir}/src/dnode/src/dnodeMain.c + # src/client/src/tscSystem.c + sed -i "s/taoslog/tqlog/g" ${top_dir}/src/client/src/tscSystem.c + # src/util/src/tnote.c + sed -i "s/taosinfo/tqinfo/g" ${top_dir}/src/util/src/tnote.c + # src/dnode/CMakeLists.txt + sed -i "s/taos\.cfg/tq\.cfg/g" ${top_dir}/src/dnode/CMakeLists.txt + # src/kit/taosdump/taosdump.c + sed -i "s/TDengine/TQueue/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/Default is taosdata/Default is tqueue/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/\"taosdata\"/\"tqueue\"/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/taos\/taos\.cfg/tq\/tq\.cfg/g" ${top_dir}/src/kit/taosdump/taosdump.c + # src/os/src/linux/linuxEnv.c + sed -i "s/etc\/taos/etc\/tq/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/lib\/taos/lib\/tq/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/log\/taos/log\/tq/g" ${top_dir}/src/os/src/linux/linuxEnv.c + # src/os/src/windows/wEnv.c + sed -i "s/TDengine/TQ/g" ${top_dir}/src/os/src/windows/wEnv.c + # src/kit/shell/src/shellEngine.c + sed -i "s/TDengine shell/TQ shell/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/2020 by TAOS Data, Inc/2020 by TQ, Inc/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\"taos> \"/\"tq> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\" -> \"/\" -> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/prompt_size = 6/prompt_size = 4/g" ${top_dir}/src/kit/shell/src/shellEngine.c +fi + +# for prodb if [[ "$dbName" == "pro" ]]; then - sed -i "s/taos config/prodb config/g" ${top_dir}/src/util/src/tconfig.c + # cmake/install.inc + sed -i "s/C:\/TDengine/C:\/ProDB/g" ${top_dir}/cmake/install.inc + sed -i "s/taos\.exe/prodbc\.exe/g" ${top_dir}/cmake/install.inc + sed -i "s/taosdemo\.exe/prodemo\.exe/g" ${top_dir}/cmake/install.inc + # src/kit/shell/inc/shell.h + sed -i "s/taos_history/prodb_history/g" ${top_dir}/src/kit/shell/inc/shell.h + # src/inc/taosdef.h + sed -i "s/\"taosdata\"/\"prodb\"/g" ${top_dir}/src/inc/taosdef.h + # src/util/src/tconfig.c + sed -i "s/taos config/prodb config/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/taos\.cfg/prodb\.cfg/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/etc\/taos/etc\/ProDB/g" ${top_dir}/src/util/src/tconfig.c + # src/kit/taosdemo/taosdemo.c + sed -i "s/\"taosdata\"/\"prodb\"/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/support@taosdata.com/support@hanatech.com.cn/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + # src/util/src/tlog.c + sed -i "s/log\/taos/log\/ProDB/g" ${top_dir}/src/util/src/tlog.c + # src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/ProDB/g" ${top_dir}/src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/ProDB/g" ${top_dir}/src/dnode/src/dnodeMain.c + sed -i "s/taosdlog/prodlog/g" ${top_dir}/src/dnode/src/dnodeMain.c + # src/client/src/tscSystem.c + sed -i "s/taoslog/prolog/g" ${top_dir}/src/client/src/tscSystem.c + # src/util/src/tnote.c + sed -i "s/taosinfo/proinfo/g" ${top_dir}/src/util/src/tnote.c + # src/dnode/CMakeLists.txt + sed -i "s/taos\.cfg/prodb\.cfg/g" ${top_dir}/src/dnode/CMakeLists.txt + # src/kit/taosdump/taosdump.c + sed -i "s/Default is taosdata/Default is prodb/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/taos\/taos\.cfg/ProDB\/prodb\.cfg/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/\"taosdata\"/\"prodb\"/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/TDengine/ProDB/g" ${top_dir}/src/kit/taosdump/taosdump.c + # src/os/src/linux/linuxEnv.c + sed -i "s/etc\/taos/etc\/ProDB/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/lib\/taos/lib\/ProDB/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/log\/taos/log\/ProDB/g" ${top_dir}/src/os/src/linux/linuxEnv.c + # src/os/src/windows/wEnv.c + sed -i "s/TDengine/ProDB/g" ${top_dir}/src/os/src/windows/wEnv.c + # src/kit/shell/src/shellEngine.c + sed -i "s/TDengine shell/ProDB shell/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/2020 by TAOS Data, Inc/2020 by Hanatech, Inc/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\"taos> \"/\"ProDB> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\" -> \"/\" -> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/prompt_size = 6/prompt_size = 7/g" ${top_dir}/src/kit/shell/src/shellEngine.c fi +# for KingHistorian +if [[ "$dbName" == "kh" ]]; then + # cmake/install.inc + sed -i "s/C:\/TDengine/C:\/KingHistorian/g" ${top_dir}/cmake/install.inc + sed -i "s/taos\.exe/khclient\.exe/g" ${top_dir}/cmake/install.inc + sed -i "s/taosdemo\.exe/khdemo\.exe/g" ${top_dir}/cmake/install.inc + # src/kit/shell/inc/shell.h + sed -i "s/taos_history/kh_history/g" ${top_dir}/src/kit/shell/inc/shell.h + # src/inc/taosdef.h + sed -i "s/\"taosdata\"/\"khroot\"/g" ${top_dir}/src/inc/taosdef.h + # src/util/src/tconfig.c + sed -i "s/taos config/kh config/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/taos\.cfg/kinghistorian\.cfg/g" ${top_dir}/src/util/src/tconfig.c + sed -i "s/etc\/taos/etc\/kinghistorian/g" ${top_dir}/src/util/src/tconfig.c + # src/kit/taosdemo/taosdemo.c + sed -i "s/taosdemo --help/khdemo --help/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/taosdemo --usage/khdemo --usage/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/Usage: taosdemo/Usage: khdemo/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/taosdemo is simulating/khdemo is simulating/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/taosdemo version/khdemo version/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/\"taosdata\"/\"khroot\"/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/support@taosdata.com/support@wellintech.com/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/taosc, rest, and stmt/khclient, rest, and stmt/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/taosdemo uses/khdemo uses/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + sed -i "s/use 'taosc'/use 'khclient'/g" ${top_dir}/src/kit/taosdemo/taosdemo.c + # src/util/src/tlog.c + sed -i "s/log\/taos/log\/kinghistorian/g" ${top_dir}/src/util/src/tlog.c + # src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/KingHistorian/g" ${top_dir}/src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/KingHistorian/g" ${top_dir}/src/dnode/src/dnodeMain.c + sed -i "s/taosdlog/khserverlog/g" ${top_dir}/src/dnode/src/dnodeMain.c + # src/client/src/tscSystem.c + sed -i "s/taoslog/khclientlog/g" ${top_dir}/src/client/src/tscSystem.c + # src/util/src/tnote.c + sed -i "s/taosinfo/khinfo/g" ${top_dir}/src/util/src/tnote.c + # src/dnode/CMakeLists.txt + sed -i "s/taos\.cfg/kinghistorian\.cfg/g" ${top_dir}/src/dnode/CMakeLists.txt + # src/kit/taosdump/taosdump.c + sed -i "s/Default is taosdata/Default is khroot/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/\"taosdata\"/\"khroot\"/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/TDengine/KingHistorian/g" ${top_dir}/src/kit/taosdump/taosdump.c + sed -i "s/taos\/taos\.cfg/kinghistorian\/kinghistorian\.cfg/g" ${top_dir}/src/kit/taosdump/taosdump.c + # src/os/src/linux/linuxEnv.c + sed -i "s/etc\/taos/etc\/kinghistorian/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/lib\/taos/lib\/kinghistorian/g" ${top_dir}/src/os/src/linux/linuxEnv.c + sed -i "s/log\/taos/log\/kinghistorian/g" ${top_dir}/src/os/src/linux/linuxEnv.c + # src/kit/shell/src/shellDarwin.c + sed -i "s/TDengine shell/KingHistorian shell/g" ${top_dir}/src/kit/shell/src/shellDarwin.c + sed -i "s/2020 by TAOS Data/2021 by Wellintech/g" ${top_dir}/src/kit/shell/src/shellDarwin.c + # src/kit/shell/src/shellLinux.c + sed -i "s/TDengine shell/KingHistorian shell/g" ${top_dir}/src/kit/shell/src/shellLinux.c + sed -i "s/2020 by TAOS Data/2021 by Wellintech/g" ${top_dir}/src/kit/shell/src/shellLinux.c + # src/os/src/windows/wEnv.c + sed -i "s/TDengine/KingHistorian/g" ${top_dir}/src/os/src/windows/wEnv.c + # src/kit/shell/src/shellEngine.c + sed -i "s/TDengine shell/KingHistorian shell/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/2020 by TAOS Data, Inc/2021 by Wellintech, Inc/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/taos connect failed/kh connect failed/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\"taos> \"/\"khclient> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/\" -> \"/\" -> \"/g" ${top_dir}/src/kit/shell/src/shellEngine.c + sed -i "s/prompt_size = 6/prompt_size = 4/g" ${top_dir}/src/kit/shell/src/shellEngine.c + # src/rpc/src/rpcMain.c + sed -i "s/taos connections/kh connections/g" ${top_dir}/src/rpc/src/rpcMain.c + # src/plugins/monitor/src/monMain.c + sed -i "s/taosd is quiting/khserver is quiting/g" ${top_dir}/src/plugins/monitor/src/monMain.c +fi + +# for jinheng +if [[ "$dbName" == "jh" ]]; then + # Following files to change: + # * src/client/src/tscSystem.c + # * src/inc/taosdef.h + # * src/kit/shell/CMakeLists.txt + # * src/kit/shell/inc/shell.h + # * src/kit/shell/src/shellEngine.c + # * src/kit/shell/src/shellWindows.c + # * src/kit/taosdemo/taosdemo.c + # * src/kit/taosdump/taosdump.c + # * src/os/src/linux/linuxEnv.c + # * src/os/src/windows/wEnv.c + # * src/util/src/tconfig.c + # * src/util/src/tlog.c + + # src/dnode/src/dnodeSystem.c + sed -i "s/TDengine/jh_iot/g" ${top_dir}/src/dnode/src/dnodeSystem.c + # src/dnode/src/dnodeMain.c + sed -i "s/TDengine/jh_iot/g" ${top_dir}/src/dnode/src/dnodeMain.c + # TODO: src/dnode/CMakeLists.txt +fi # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then if [ "$verMode" != "cluster" ]; then + # community-version compile cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} ${allocator_macro} else + # enterprise-version compile + if [[ "$dbName" == "power" ]]; then + # enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/\"taosdata\"/\"powerdb\"/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/TDengine/PowerDB/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + # enterprise/src/plugins/admin/src/httpAdminHandle.c + sed -i "s/taos\.cfg/power\.cfg/g" ${top_dir}/../enterprise/src/plugins/admin/src/httpAdminHandle.c + # enterprise/src/plugins/grant/src/grantMain.c + sed -i "s/taos\.cfg/power\.cfg/g" ${top_dir}/../enterprise/src/plugins/grant/src/grantMain.c + # enterprise/src/plugins/module/src/moduleMain.c + sed -i "s/taos\.cfg/power\.cfg/g" ${top_dir}/../enterprise/src/plugins/module/src/moduleMain.c + fi + if [[ "$dbName" == "tq" ]]; then + # enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/\"taosdata\"/\"tqueue\"/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/TDengine/TQueue/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + # enterprise/src/plugins/admin/src/httpAdminHandle.c + sed -i "s/taos\.cfg/tq\.cfg/g" ${top_dir}/../enterprise/src/plugins/admin/src/httpAdminHandle.c + # enterprise/src/plugins/grant/src/grantMain.c + sed -i "s/taos\.cfg/tq\.cfg/g" ${top_dir}/../enterprise/src/plugins/grant/src/grantMain.c + # enterprise/src/plugins/module/src/moduleMain.c + sed -i "s/taos\.cfg/tq\.cfg/g" ${top_dir}/../enterprise/src/plugins/module/src/moduleMain.c + fi + if [[ "$dbName" == "pro" ]]; then + # enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/\"taosdata\"/\"prodb\"/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/TDengine/ProDB/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + # enterprise/src/plugins/admin/src/httpAdminHandle.c + sed -i "s/taos\.cfg/prodb\.cfg/g" ${top_dir}/../enterprise/src/plugins/admin/src/httpAdminHandle.c + # enterprise/src/plugins/grant/src/grantMain.c + sed -i "s/taos\.cfg/prodb\.cfg/g" ${top_dir}/../enterprise/src/plugins/grant/src/grantMain.c + # enterprise/src/plugins/module/src/moduleMain.c + sed -i "s/taos\.cfg/prodb\.cfg/g" ${top_dir}/../enterprise/src/plugins/module/src/moduleMain.c + fi + if [[ "$dbName" == "kh" ]]; then + # enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/\"taosdata\"/\"khroot\"/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/TDengine/KingHistorian/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + # enterprise/src/plugins/admin/src/httpAdminHandle.c + sed -i "s/taos\.cfg/kinghistorian\.cfg/g" ${top_dir}/../enterprise/src/plugins/admin/src/httpAdminHandle.c + # enterprise/src/plugins/grant/src/grantMain.c + sed -i "s/taos\.cfg/kinghistorian\.cfg/g" ${top_dir}/../enterprise/src/plugins/grant/src/grantMain.c + # enterprise/src/plugins/module/src/moduleMain.c + sed -i "s/taos\.cfg/kinghistorian\.cfg/g" ${top_dir}/../enterprise/src/plugins/module/src/moduleMain.c + fi + if [[ "$dbName" == "jh" ]]; then + # enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/\"taosdata\"/\"jhdata\"/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + sed -i "s/TDengine/jh_iot/g" ${top_dir}/../enterprise/src/kit/perfMonitor/perfMonitor.c + # enterprise/src/plugins/admin/src/httpAdminHandle.c + #sed -i "s/taos\.cfg/taos\.cfg/g" ${top_dir}/../enterprise/src/plugins/admin/src/httpAdminHandle.c + # enterprise/src/plugins/grant/src/grantMain.c + #sed -i "s/taos\.cfg/taos\.cfg/g" ${top_dir}/../enterprise/src/plugins/grant/src/grantMain.c + # enterprise/src/plugins/module/src/moduleMain.c + #sed -i "s/taos\.cfg/taos\.cfg/g" ${top_dir}/../enterprise/src/plugins/module/src/moduleMain.c + fi + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} ${allocator_macro} fi else @@ -230,7 +515,6 @@ if [ "$osType" != "Darwin" ]; then else echo "==========dpkg command not exist, so not release deb package!!!" fi - ret='0' command -v rpmbuild >/dev/null 2>&1 || { ret='1'; } if [ "$ret" -eq 0 ]; then @@ -262,12 +546,21 @@ if [ "$osType" != "Darwin" ]; then ${csudo} ./makepkg_pro.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${verNumberComp} ${csudo} ./makeclient_pro.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${csudo} ./makearbi_pro.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + elif [[ "$dbName" == "kh" ]]; then + ${csudo} ./makepkg_kh.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${verNumberComp} + ${csudo} ./makeclient_kh.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makearbi_kh.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + elif [[ "$dbName" == "jh" ]]; then + ${csudo} ./makepkg_jh.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${verNumberComp} + ${csudo} ./makeclient_jh.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makearbi_jh.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else ${csudo} ./makepkg_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${verNumberComp} ${csudo} ./makeclient_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${csudo} ./makearbi_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} fi else + # only make client for Darwin cd ${script_dir}/tools ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${dbName} fi diff --git a/packaging/rpm/nginxd b/packaging/rpm/nginxd new file mode 100644 index 0000000000000000000000000000000000000000..fc6593992b297f7fcad03d04051a030dc1898d5e --- /dev/null +++ b/packaging/rpm/nginxd @@ -0,0 +1,44 @@ +#!/bin/bash +#Startup script for the nginx Web Server +# chkconfig: 2345 99 01 +# +# +### BEGIN INIT INFO +# Provides: nginx +# Provides: nginx +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Short-Description: Starts nginx +# Description: Starts nginx. +### END INIT INFO + +nginx=/usr/local/nginxd/sbin/nginx +case $1 in + start) + echo -n "Starting Nginx" + $nginx + echo " done." + ;; + stop) + echo -n "Stopping Nginx" + $nginx -s stop + echo " done." + ;; + test) + $nginx -t + echo "Success." + ;; + reload) + echo -n "Reloading Nginx" + $nginx -s reload + echo " done." + ;; + restart) + $nginx -s reload + echo "reload done." + ;; + *) + echo "Usage: $0 {start|restart|reload|stop|test|show}" + ;; +esac + diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 8a870286aba1793ec880af6dd0d8a21602ddc86e..551f56bf0cf850a3680e8e92d36ecdcda737de49 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -67,12 +67,6 @@ cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include -if [ -d %{_compiledir}/../src/connector/grafanaplugin/dist ]; then - cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin -else - echo grafanaplugin bundled directory not found! - exit 1 -fi cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 0d7a7dd9442fd635fce56792dc37c3276d1b955d..3930433b96166c8beb0bddca17aa73ab11c0ebc5 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -582,6 +582,26 @@ function clean_service_on_sysvinit() { ${csudo} rm -f ${service_config_dir}/taosd || : ${csudo} rm -f ${service_config_dir}/tarbitratord || : + if [ "$verMode" == "cluster" ]; then\ + if pidof nginxd ; then + ${csudo} service nginxd stop || : + fi + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/nginxd ]; then + ${csudo} chkconfig --del nginxd || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/nginxd ]; then + ${csudo} insserv -r nginxd || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/nginxd ]; then + ${csudo} update-rc.d -f nginxd remove || : + fi + fi + ${csudo} rm -f ${service_config_dir}/nginxd || : + fi + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -598,11 +618,19 @@ function install_service_on_sysvinit() { ${csudo} cp ${script_dir}/init.d/taosd.deb ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -f ${script_dir}/init.d/nginxd.deb ${install_main_dir}/init.d/nginxd + ${csudo} cp ${script_dir}/init.d/nginxd.deb ${service_config_dir}/nginxd && ${csudo} chmod a+x ${service_config_dir}/nginxd + fi elif ((${os_type}==2)); then ${csudo} cp -f ${script_dir}/init.d/taosd.rpm ${install_main_dir}/init.d/taosd ${csudo} cp ${script_dir}/init.d/taosd.rpm ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -f ${script_dir}/init.d/nginxd.rpm ${install_main_dir}/init.d/nginxd + ${csudo} cp ${script_dir}/init.d/nginxd.rpm ${service_config_dir}/nginxd && ${csudo} chmod a+x ${service_config_dir}/nginxd + fi fi #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" @@ -613,14 +641,26 @@ function install_service_on_sysvinit() { ${csudo} chkconfig --level 2345 taosd on || : ${csudo} chkconfig --add tarbitratord || : ${csudo} chkconfig --level 2345 tarbitratord on || : + if [ "$verMode" == "cluster" ]; then + ${csudo} chkconfig --add nginxd || : + ${csudo} chkconfig --level 0123456 nginxd on || : + ${csudo} service nginxd start + fi elif ((${initd_mod}==2)); then ${csudo} insserv taosd || : ${csudo} insserv -d taosd || : ${csudo} insserv tarbitratord || : ${csudo} insserv -d tarbitratord || : + if [ "$verMode" == "cluster" ]; then + ${csudo} insserv nginxd || : + ${csudo} insserv -d nginxd || : + fi elif ((${initd_mod}==3)); then ${csudo} update-rc.d taosd defaults || : ${csudo} update-rc.d tarbitratord defaults || : + if [ "$verMode" == "cluster" ]; then + ${csudo} update-rc.d nginxd defaults || : + fi fi } @@ -779,7 +819,7 @@ vercomp () { function is_version_compatible() { - curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` + curr_version=`ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}'` if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` diff --git a/packaging/tools/install_arbi_jh.sh b/packaging/tools/install_arbi_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..2403f8fbd79abf4324577fe3dca3a8e0eac8ed01 --- /dev/null +++ b/packaging/tools/install_arbi_jh.sh @@ -0,0 +1,286 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact jhict.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi_jh.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi_jh.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install server service + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=jh_iot arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + kill_tarbitrator + fi +} + +function update() { + # Start to update + echo -e "${GREEN}Start to update jh_iot's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mjh_iot's arbitrator is updated successfully!${NC}" +} + +function install() { + # Start to install + echo -e "${GREEN}Start to install jh_iot's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mjh_iot's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update +else + install +fi + diff --git a/packaging/tools/install_arbi_kh.sh b/packaging/tools/install_arbi_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..9a2542936d935b70b762702f0f2f6ff92b51a4f3 --- /dev/null +++ b/packaging/tools/install_arbi_kh.sh @@ -0,0 +1,286 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact wellintech.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi_kh.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi_kh.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install khserver service + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=KingHistorian arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + kill_tarbitrator + fi +} + +function update() { + # Start to update + echo -e "${GREEN}Start to update KingHistorian's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mKingHistorian's arbitrator is updated successfully!${NC}" +} + +function install() { + # Start to install + echo -e "${GREEN}Start to install KingHistorian's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mKingHistorian's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update +else + install +fi + diff --git a/packaging/tools/install_arbi_power.sh b/packaging/tools/install_arbi_power.sh index 883db2b7169d125309125887cb72279c92c4602a..da86566eec43438cc9f266a86f4e7afe226ca6ce 100755 --- a/packaging/tools/install_arbi_power.sh +++ b/packaging/tools/install_arbi_power.sh @@ -176,9 +176,6 @@ function install_header() { } function clean_service_on_sysvinit() { - #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi @@ -279,7 +276,6 @@ function install_service() { elif ((${service_mod}==1)); then install_service_on_sysvinit else - # must manual stop taosd kill_tarbitrator fi } @@ -306,7 +302,6 @@ function update_PowerDB() { install_jemalloc echo - #echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" elif ((${service_mod}==1)); then @@ -329,7 +324,6 @@ function install_PowerDB() { install_jemalloc echo - #echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" elif ((${service_mod}==1)); then diff --git a/packaging/tools/install_arbi_pro.sh b/packaging/tools/install_arbi_pro.sh index 11165dbdd8bdf6afb4659250499cf1d9184c2395..98e896aa6583485ae1700d3557b0fe93a34b0a47 100755 --- a/packaging/tools/install_arbi_pro.sh +++ b/packaging/tools/install_arbi_pro.sh @@ -123,9 +123,6 @@ function install_header() { } function clean_service_on_sysvinit() { - #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi @@ -221,7 +218,6 @@ function install_service() { elif ((${service_mod}==1)); then install_service_on_sysvinit else - # must manual stop taosd kill_tarbitrator fi } @@ -247,7 +243,6 @@ function update_prodb() { install_service echo - #echo -e "${GREEN_DARK}To configure ProDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" elif ((${service_mod}==1)); then @@ -268,7 +263,6 @@ function install_prodb() { install_bin install_service echo - #echo -e "${GREEN_DARK}To configure ProDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" elif ((${service_mod}==1)); then diff --git a/packaging/tools/install_arbi_tq.sh b/packaging/tools/install_arbi_tq.sh index bd852dd0ad2c9114f2424193adccf56b0cb40412..37426b359d8957c664fd21552a95d45f50085485 100755 --- a/packaging/tools/install_arbi_tq.sh +++ b/packaging/tools/install_arbi_tq.sh @@ -123,9 +123,6 @@ function install_header() { } function clean_service_on_sysvinit() { - #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi @@ -226,7 +223,6 @@ function install_service() { elif ((${service_mod}==1)); then install_service_on_sysvinit else - # must manual stop taosd kill_tarbitrator fi } @@ -252,7 +248,6 @@ function update_tq() { install_service echo - #echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" elif ((${service_mod}==1)); then @@ -273,7 +268,6 @@ function install_tq() { install_bin install_service echo - #echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" elif ((${service_mod}==1)); then diff --git a/packaging/tools/install_client_jh.sh b/packaging/tools/install_client_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..a8599812feaea06120aa3391771e7c94523a53c2 --- /dev/null +++ b/packaging/tools/install_client_jh.sh @@ -0,0 +1,239 @@ +#!/bin/bash +# +# This file is used to install jh_taos client on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- + +osType=Linux +pagMode=full + +if [ "$osType" != "Darwin" ]; then + script_dir=$(dirname $(readlink -f "$0")) + # Dynamic directory + data_dir="/var/lib/jh_taos" + log_dir="/var/log/jh_taos" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + data_dir="/var/lib/jh_taos" + log_dir="~/jh_taos/log" +fi + +log_link_dir="/usr/local/jh_taos/log" + +cfg_install_dir="/etc/jh_taos" + +if [ "$osType" != "Darwin" ]; then + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" +else + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" +fi + +#install main path +install_main_dir="/usr/local/jh_taos" + +# old bin dir +bin_dir="/usr/local/jh_taos/bin" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +function kill_client() { + pid=$(ps -ef | grep "jh_taos" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/jh_taos || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${bin_link_dir}/jhdemo || : + ${csudo} rm -f ${bin_link_dir}/jh_taosdump || : + fi + ${csudo} rm -f ${bin_link_dir}/rmjh || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/jh_taos ] && ${csudo} ln -s ${install_main_dir}/bin/jh_taos ${bin_link_dir}/jh_taos || : + if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/jhdemo ] && ${csudo} ln -s ${install_main_dir}/bin/jhdemo ${bin_link_dir}/jhdemo || : + [ -x ${install_main_dir}/bin/jh_taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/jh_taosdump ${bin_link_dir}/jh_taosdump || : + fi + [ -x ${install_main_dir}/bin/remove_client_jh.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_jh.sh ${bin_link_dir}/rmjh || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : +} + +function clean_lib() { + sudo rm -f /usr/lib/libtaos.* || : + sudo rm -rf ${lib_dir} || : +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [ -d "${lib64_link_dir}" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + else + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + fi + + ${csudo} ldconfig +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_config() { + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + + if [ "$osType" != "Darwin" ]; then + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + else + mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + fi + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function update() { + # Start to update + if [ ! -e jh_taos.tar.gz ]; then + echo "File jh_taos.tar.gz does not exist" + exit 1 + fi + tar -zxf jh_taos.tar.gz + + echo -e "${GREEN}Start to update jh_iot client...${NC}" + # Stop the client shell if running + if pidof jh_taos &> /dev/null; then + kill_client + sleep 1 + fi + + install_main_path + + install_log + install_header + install_lib + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mjh_iot client is updated successfully!${NC}" + + rm -rf $(tar -tf jh_taos.tar.gz) +} + +function install() { + # Start to install + if [ ! -e jh_taos.tar.gz ]; then + echo "File jh_taos.tar.gz does not exist" + exit 1 + fi + tar -zxf jh_taos.tar.gz + + echo -e "${GREEN}Start to install jh_taos client...${NC}" + + install_main_path + install_log + install_header + install_lib + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mjh_iot client is installed successfully!${NC}" + + rm -rf $(tar -tf jh_taos.tar.gz) +} + + +## ==============================Main program starts from here============================ +# Install or updata client and client +# if server is already install, don't install client + if [ -e ${bin_dir}/jh_taosd ]; then + echo -e "\033[44;32;1mThere are already installed jh_iot server, so don't need install client!${NC}" + exit 0 + fi + + if [ -x ${bin_dir}/jh_taos ]; then + update_flag=1 + update + else + install + fi diff --git a/packaging/tools/install_client_kh.sh b/packaging/tools/install_client_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..3e7df18486a20b9ea75dba3d2644d46bee6b423b --- /dev/null +++ b/packaging/tools/install_client_kh.sh @@ -0,0 +1,240 @@ +#!/bin/bash +# +# This file is used to install kinghistorian client on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- + +osType=Linux +pagMode=full + +if [ "$osType" != "Darwin" ]; then + script_dir=$(dirname $(readlink -f "$0")) + # Dynamic directory + data_dir="/var/lib/kinghistorian" + log_dir="/var/log/kinghistorian" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + data_dir="/var/lib/kinghistorian" + log_dir="~/kinghistorian/log" +fi + +log_link_dir="/usr/local/kinghistorian/log" + +cfg_install_dir="/etc/kinghistorian" + +if [ "$osType" != "Darwin" ]; then + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" +else + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" +fi + +#install main path +install_main_dir="/usr/local/kinghistorian" + +# old bin dir +bin_dir="/usr/local/kinghistorian/bin" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +function kill_client() { + pid=$(ps -ef | grep "khclient" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/khclient || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${bin_link_dir}/khdemo || : + ${csudo} rm -f ${bin_link_dir}/khdump || : + fi + ${csudo} rm -f ${bin_link_dir}/rmkh || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/khclient ] && ${csudo} ln -s ${install_main_dir}/bin/khclient ${bin_link_dir}/khclient || : + if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/khdemo ] && ${csudo} ln -s ${install_main_dir}/bin/khdemo ${bin_link_dir}/khdemo || : + [ -x ${install_main_dir}/bin/khdump ] && ${csudo} ln -s ${install_main_dir}/bin/khdump ${bin_link_dir}/khdump || : + fi + [ -x ${install_main_dir}/bin/remove_client_kh.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_kh.sh ${bin_link_dir}/rmkh || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : +} + +function clean_lib() { + sudo rm -f /usr/lib/libtaos.* || : + sudo rm -rf ${lib_dir} || : +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : + + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [ -d "${lib64_link_dir}" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + else + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + fi + + ${csudo} ldconfig +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_config() { + if [ ! -f ${cfg_install_dir}/kinghistorian.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/kinghistorian.cfg ] && ${csudo} cp ${script_dir}/cfg/kinghistorian.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/kinghistorian.cfg ${install_main_dir}/cfg/kinghistorian.cfg.org + ${csudo} ln -s ${cfg_install_dir}/kinghistorian.cfg ${install_main_dir}/cfg +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + + if [ "$osType" != "Darwin" ]; then + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + else + mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + fi + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function update() { + # Start to update + if [ ! -e kinghistorian.tar.gz ]; then + echo "File kinghistorian.tar.gz does not exist" + exit 1 + fi + tar -zxf kinghistorian.tar.gz + + echo -e "${GREEN}Start to update KingHistorian client...${NC}" + # Stop the client shell if running + if pidof khclient &> /dev/null; then + kill_client + sleep 1 + fi + + install_main_path + + install_log + install_header + install_lib + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mKingHistorian client is updated successfully!${NC}" + + rm -rf $(tar -tf kinghistorian.tar.gz) +} + +function install() { + # Start to install + if [ ! -e kinghistorian.tar.gz ]; then + echo "File kinghistorian.tar.gz does not exist" + exit 1 + fi + tar -zxf kinghistorian.tar.gz + + echo -e "${GREEN}Start to install KingHistorian client...${NC}" + + install_main_path + install_log + install_header + install_lib + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mKingHistorian client is installed successfully!${NC}" + + rm -rf $(tar -tf kinghistorian.tar.gz) +} + + +## ==============================Main program starts from here============================ +# Install or updata client and client +# if server is already install, don't install client + if [ -e ${bin_dir}/khserver ]; then + echo -e "\033[44;32;1mThere are already installed KingHistorian server, so don't need install client!${NC}" + exit 0 + fi + + if [ -x ${bin_dir}/khclient ]; then + update_flag=1 + update + else + install + fi diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 31da0d61319045800fe3a454d071118aa3a4768e..64c355f0074dd158052d57e1df7da220c6614586 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -193,16 +193,14 @@ function install_jemalloc() { } function install_config() { - #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + if [ ! -f ${cfg_install_dir}/power.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/cfg/power.cfg ] && ${csudo} cp ${script_dir}/cfg/power.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi - ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} cp -f ${script_dir}/cfg/power.cfg ${install_main_dir}/cfg/power.cfg.org + ${csudo} ln -s ${cfg_install_dir}/power.cfg ${install_main_dir}/cfg } @@ -248,9 +246,6 @@ function update_PowerDB() { install_log install_header install_lib - if [ "$pagMode" != "lite" ]; then - install_connector - fi install_examples install_bin install_config @@ -276,9 +271,6 @@ function install_PowerDB() { install_header install_lib install_jemalloc - if [ "$pagMode" != "lite" ]; then - install_connector - fi install_examples install_bin install_config diff --git a/packaging/tools/install_client_pro.sh b/packaging/tools/install_client_pro.sh index fff8ae31200669ee3ab918a873e33fc32ece37c8..b21c90b7aa5bf9348f891222fecbdc04f8c72f87 100755 --- a/packaging/tools/install_client_pro.sh +++ b/packaging/tools/install_client_pro.sh @@ -109,7 +109,6 @@ function install_lib() { # Remove links ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : - #${csudo} rm -rf ${v15_java_app_dir} || : ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* @@ -137,16 +136,14 @@ function install_header() { } function install_config() { - #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + if [ ! -f ${cfg_install_dir}/prodb.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/cfg/prodb.cfg ] && ${csudo} cp ${script_dir}/cfg/prodb.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi - ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} cp -f ${script_dir}/cfg/prodb.cfg ${install_main_dir}/cfg/prodb.cfg.org + ${csudo} ln -s ${cfg_install_dir}/prodb.cfg ${install_main_dir}/cfg } @@ -191,9 +188,6 @@ function update_prodb() { install_log install_header install_lib - if [ "$pagMode" != "lite" ]; then - install_connector - fi install_examples install_bin install_config @@ -218,9 +212,6 @@ function install_prodb() { install_log install_header install_lib - if [ "$pagMode" != "lite" ]; then - install_connector - fi install_examples install_bin install_config @@ -235,14 +226,14 @@ function install_prodb() { ## ==============================Main program starts from here============================ # Install or updata client and client # if server is already install, don't install client - if [ -e ${bin_dir}/prodbs ]; then - echo -e "\033[44;32;1mThere are already installed ProDB server, so don't need install client!${NC}" - exit 0 - fi +if [ -e ${bin_dir}/prodbs ]; then + echo -e "\033[44;32;1mThere are already installed ProDB server, so don't need install client!${NC}" + exit 0 +fi - if [ -x ${bin_dir}/prodbc ]; then - update_flag=1 - update_prodb - else - install_prodb - fi +if [ -x ${bin_dir}/prodbc ]; then + update_flag=1 + update_prodb +else + install_prodb +fi diff --git a/packaging/tools/install_client_tq.sh b/packaging/tools/install_client_tq.sh index 2537442ee264e9aeb4eb6b3d25a17faf60f4df9a..119b87fe3ee496c11562c44f223fbef8cf302402 100755 --- a/packaging/tools/install_client_tq.sh +++ b/packaging/tools/install_client_tq.sh @@ -140,16 +140,14 @@ function install_header() { } function install_config() { - #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + if [ ! -f ${cfg_install_dir}/tq.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/cfg/tq.cfg ] && ${csudo} cp ${script_dir}/cfg/tq.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi - ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} cp -f ${script_dir}/cfg/tq.cfg ${install_main_dir}/cfg/tq.cfg.org + ${csudo} ln -s ${cfg_install_dir}/tq.cfg ${install_main_dir}/cfg } @@ -194,9 +192,6 @@ function update_tq() { install_log install_header install_lib - if [ "$pagMode" != "lite" ]; then - install_connector - fi install_examples install_bin install_config @@ -221,9 +216,6 @@ function install_tq() { install_log install_header install_lib - if [ "$pagMode" != "lite" ]; then - install_connector - fi install_examples install_bin install_config diff --git a/packaging/tools/install_jh.sh b/packaging/tools/install_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..867c5c93b647b788f5ef48c8ea3e6172747cacab --- /dev/null +++ b/packaging/tools/install_jh.sh @@ -0,0 +1,948 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) +# Dynamic directory +data_dir="/var/lib/jh_taos" +log_dir="/var/log/jh_taos" + +data_link_dir="/usr/local/jh_taos/data" +log_link_dir="/usr/local/jh_taos/log" + +cfg_install_dir="/etc/jh_taos" + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/jh_taos" + +# old bin dir +bin_dir="/usr/local/jh_taos/bin" + +service_config_dir="/etc/systemd/system" +nginx_port=6060 +nginx_dir="/usr/local/nginxd" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact jhict.com for support." + os_type=1 +fi + + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:e:i:" arg +do + case $arg in + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$( echo $OPTARG ) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + i) + #echo "initType=$OPTARG" + initType=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin +# ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver +# ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d + if [ "$verMode" == "cluster" ]; then + ${csudo} mkdir -p ${nginx_dir} + fi +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/jh_taos || : + ${csudo} rm -f ${bin_link_dir}/jh_taosd || : + ${csudo} rm -f ${bin_link_dir}/jhdemo || : + ${csudo} rm -f ${bin_link_dir}/rmjh || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/jh_taos ] && ${csudo} ln -s ${install_main_dir}/bin/jh_taos ${bin_link_dir}/jh_taos || : + [ -x ${install_main_dir}/bin/jh_taosd ] && ${csudo} ln -s ${install_main_dir}/bin/jh_taosd ${bin_link_dir}/jh_taosd || : + [ -x ${install_main_dir}/bin/jhdemo ] && ${csudo} ln -s ${install_main_dir}/bin/jhdemo ${bin_link_dir}/jhdemo || : + [ -x ${install_main_dir}/bin/remove_jh.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_jh.sh ${bin_link_dir}/rmjh || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : + + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo} chmod 0555 ${nginx_dir}/* + ${csudo} mkdir -p ${nginx_dir}/logs + ${csudo} chmod 777 ${nginx_dir}/sbin/nginx + fi +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + + if [ "$osType" != "Darwin" ]; then + ${csudo} ldconfig + else + ${csudo} update_dyld_shared_cache + fi +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + +function install_config() { + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + + [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag}==1)); then + return 0 + fi + + if [ "$interactiveFqdn" == "no" ]; then + return 0 + fi + + local_fqdn_check + + #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" + #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" + #PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)" + #FQDN_PATTERN=":[0-9]{1,5}$" + + # first full-qualified domain name (FQDN) for jh_iot cluster system + echo + echo -e -n "${GREEN}Enter FQDN:port (like h1.jhict.com:6030) of an existing jh_iot cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" + read firstEp + while true; do + if [ ! -z "$firstEp" ]; then + # check the format of the firstEp + #if [[ $firstEp == $FQDN_PATTERN ]]; then + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + break + #else + # read -p "Please enter the correct FQDN:port: " firstEp + #fi + else + break + fi + done +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_data() { + ${csudo} mkdir -p ${data_dir} + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function clean_service_on_sysvinit() { + if pidof jh_taosd &> /dev/null; then + ${csudo} service jh_taosd stop || : + fi + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/jh_taosd ]; then + ${csudo} chkconfig --del jh_taosd || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/jh_taosd ]; then + ${csudo} insserv -r jh_taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/jh_taosd ]; then + ${csudo} update-rc.d -f jh_taosd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/jh_taosd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install jh_taosd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/jh_taosd.deb ${install_main_dir}/init.d/jh_taosd + ${csudo} cp ${script_dir}/init.d/jh_taosd.deb ${service_config_dir}/jh_taosd && ${csudo} chmod a+x ${service_config_dir}/jh_taosd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/jh_taosd.rpm ${install_main_dir}/init.d/jh_taosd + ${csudo} cp ${script_dir}/init.d/jh_taosd.rpm ${service_config_dir}/jh_taosd && ${csudo} chmod a+x ${service_config_dir}/jh_taosd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add jh_taosd || : + ${csudo} chkconfig --level 2345 jh_taosd on || : + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv jh_taosd || : + ${csudo} insserv -d jh_taosd || : + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d jh_taosd defaults || : + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + jh_taosd_service_config="${service_config_dir}/jh_taosd.service" + if systemctl is-active --quiet jh_taosd; then + echo "jh_iot is running, stopping it..." + ${csudo} systemctl stop jh_taosd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable jh_taosd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${jh_taosd_service_config} + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + if systemctl is-active --quiet nginxd; then + echo "Nginx for jh_iot is running, stopping it..." + ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${nginx_service_config} + fi +} + +function install_service_on_systemd() { + clean_service_on_systemd + + service_config="${service_config_dir}/jh_taosd.service" + ${csudo} bash -c "echo '[Unit]' >> ${service_config}" + ${csudo} bash -c "echo 'Description=jh_iot server service' >> ${service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${service_config}" + ${csudo} bash -c "echo >> ${service_config}" + ${csudo} bash -c "echo '[Service]' >> ${service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/jh_taosd' >> ${service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/jh_taos/bin/startPre.sh' >> ${service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${service_config}" + ${csudo} bash -c "echo >> ${service_config}" + ${csudo} bash -c "echo '[Install]' >> ${service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${service_config}" + ${csudo} systemctl enable jh_taosd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=jh_iot arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Description=Nginx For jh_iot Service' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Type=forking' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${nginx_service_config}" + if ! ${csudo} systemctl enable nginxd &> /dev/null; then + ${csudo} systemctl daemon-reexec + ${csudo} systemctl enable nginxd + fi + ${csudo} systemctl start nginxd + fi +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop jh_taosd + kill_process jh_taosd + fi +} + +vercomp () { + if [[ $1 == $2 ]]; then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +function is_version_compatible() { + curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` + + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` + else + min_compatible_version=$(${script_dir}/bin/jh_taosd -V | head -1 | cut -d ' ' -f 5) + fi + + vercomp $curr_version $min_compatible_version + case $? in + 0) return 0;; + 1) return 0;; + 2) return 1;; + esac +} + +function update() { + # Start to update + if [ ! -e jh_taos.tar.gz ]; then + echo "File jh_taos.tar.gz does not exist" + exit 1 + fi + tar -zxf jh_taos.tar.gz + install_jemalloc + + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + + echo -e "${GREEN}Start to update jh_iot...${NC}" + # Stop the service if running + if pidof jh_taosd &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop jh_taosd || : + elif ((${service_mod}==1)); then + ${csudo} service jh_taosd stop || : + else + kill_process jh_taosd + fi + sleep 1 + fi + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx + fi + sleep 1 + fi + fi + + install_main_path + + install_log + install_header + install_lib +# if [ "$pagMode" != "lite" ]; then +# install_connector +# fi +# install_examples + if [ -z $1 ]; then + install_bin + install_service + install_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if openresty is installed + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for jh_iot is updated successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for jh_iot does not work! Please try again!\033[0m" + fi + fi + fi + + #echo + #echo -e "\033[44;32;1mjh_iot is updated successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure jh_iot ${NC}: edit /etc/jh_taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start jh_iot ${NC}: ${csudo} systemctl start jh_taosd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start jh_iot ${NC}: ${csudo} service jh_taosd start${NC}" + else + echo -e "${GREEN_DARK}To start jh_iot ${NC}: ./jh_taosd${NC}" + fi + + if [ ${openresty_work} = 'true' ]; then + echo -e "${GREEN_DARK}To access jh_iot ${NC}: use ${GREEN_UNDERLINE}jh_taos -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + else + echo -e "${GREEN_DARK}To access jh_iot ${NC}: use ${GREEN_UNDERLINE}jh_taos -h $serverFqdn${NC} in shell${NC}" + fi + + echo + echo -e "\033[44;32;1mjh_iot is updated successfully!${NC}" + else + install_bin + install_config + + echo + echo -e "\033[44;32;1mjh_iot client is updated successfully!${NC}" + fi + + rm -rf $(tar -tf jh_taos.tar.gz) +} + +function install() { + # Start to install + if [ ! -e jh_taos.tar.gz ]; then + echo "File jh_taos.tar.gz does not exist" + exit 1 + fi + tar -zxf jh_taos.tar.gz + + echo -e "${GREEN}Start to install jh_iot...${NC}" + + install_main_path + + if [ -z $1 ]; then + install_data + fi + + install_log + install_header + install_lib + install_jemalloc +# if [ "$pagMode" != "lite" ]; then +# install_connector +# fi +# install_examples + + if [ -z $1 ]; then # install service and client + # For installing new + install_bin + install_service + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for jh_iot is installed successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for jh_iot does not work! Please try again!\033[0m" + fi + fi + fi + + install_config + + # Ask if to start the service + #echo + #echo -e "\033[44;32;1mjh_iot is installed successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure jh_iot ${NC}: edit /etc/jh_taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start jh_iot ${NC}: ${csudo} systemctl start jh_taosd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start jh_iot ${NC}: ${csudo} service jh_taosd start${NC}" + else + echo -e "${GREEN_DARK}To start jh_iot ${NC}: jh_taosd${NC}" + fi + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access jh_iot ${NC}: jh_taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access jh_iot ${NC}: jh_taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access jh_iot ${NC}: jh_taos -h $serverFqdn${GREEN_DARK} to login into jh_iot server${NC}" + echo + fi + echo -e "\033[44;32;1mjh_iot is installed successfully!${NC}" + echo + else # Only install client + install_bin + install_config + + echo + echo -e "\033[44;32;1mjh_iot client is installed successfully!${NC}" + fi + + rm -rf $(tar -tf jh_taos.tar.gz) +} + + +## ==============================Main program starts from here============================ +serverFqdn=$(hostname) +if [ "$verType" == "server" ]; then + # Install server and client + if [ -x ${bin_dir}/jh_taosd ]; then + update_flag=1 + update + else + install + fi +elif [ "$verType" == "client" ]; then + interactiveFqdn=no + # Only install client + if [ -x ${bin_dir}/jh_taos ]; then + update_flag=1 + update client + else + install client + fi +else + echo "please input correct verType" +fi diff --git a/packaging/tools/install_kh.sh b/packaging/tools/install_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..4ec57dcee50fb46246b961e0f3bd3751fec08cae --- /dev/null +++ b/packaging/tools/install_kh.sh @@ -0,0 +1,950 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) +# Dynamic directory +data_dir="/var/lib/kinghistorian" +log_dir="/var/log/kinghistorian" + +data_link_dir="/usr/local/kinghistorian/data" +log_link_dir="/usr/local/kinghistorian/log" + +cfg_install_dir="/etc/kinghistorian" + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/kinghistorian" + +# old bin dir +bin_dir="/usr/local/kinghistorian/bin" + +service_config_dir="/etc/systemd/system" +nginx_port=6060 +nginx_dir="/usr/local/nginxd" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact wellintech.com for support." + os_type=1 +fi + + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:e:i:" arg +do + case $arg in + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$( echo $OPTARG ) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + i) + #echo "initType=$OPTARG" + initType=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin +# ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver +# ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d + if [ "$verMode" == "cluster" ]; then + ${csudo} mkdir -p ${nginx_dir} + fi +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/khclient || : + ${csudo} rm -f ${bin_link_dir}/khserver || : + ${csudo} rm -f ${bin_link_dir}/khdemo || : + ${csudo} rm -f ${bin_link_dir}/khdump || : + ${csudo} rm -f ${bin_link_dir}/rmkh || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/khclient ] && ${csudo} ln -s ${install_main_dir}/bin/khclient ${bin_link_dir}/khclient || : + [ -x ${install_main_dir}/bin/khserver ] && ${csudo} ln -s ${install_main_dir}/bin/khserver ${bin_link_dir}/khserver || : + [ -x ${install_main_dir}/bin/khdump ] && ${csudo} ln -s ${install_main_dir}/bin/khdump ${bin_link_dir}/khdump || : + [ -x ${install_main_dir}/bin/khdemo ] && ${csudo} ln -s ${install_main_dir}/bin/khdemo ${bin_link_dir}/khdemo || : + [ -x ${install_main_dir}/bin/remove_kh.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_kh.sh ${bin_link_dir}/rmkh || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : + + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo} chmod 0555 ${nginx_dir}/* + ${csudo} mkdir -p ${nginx_dir}/logs + ${csudo} chmod 777 ${nginx_dir}/sbin/nginx + fi +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + + if [ "$osType" != "Darwin" ]; then + ${csudo} ldconfig + else + ${csudo} update_dyld_shared_cache + fi +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/kinghistorian.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/kinghistorian.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/kinghistorian.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + +function install_config() { + if [ ! -f ${cfg_install_dir}/kinghistorian.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/kinghistorian.cfg ] && ${csudo} cp ${script_dir}/cfg/kinghistorian.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/kinghistorian.cfg ${install_main_dir}/cfg/kinghistorian.cfg.org + ${csudo} ln -s ${cfg_install_dir}/kinghistorian.cfg ${install_main_dir}/cfg + + [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag}==1)); then + return 0 + fi + + if [ "$interactiveFqdn" == "no" ]; then + return 0 + fi + + local_fqdn_check + + #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" + #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" + #PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)" + #FQDN_PATTERN=":[0-9]{1,5}$" + + # first full-qualified domain name (FQDN) for KingHistorian cluster system + echo + echo -e -n "${GREEN}Enter FQDN:port (like h1.wellintech.com:6030) of an existing KingHistorian cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" + read firstEp + while true; do + if [ ! -z "$firstEp" ]; then + # check the format of the firstEp + #if [[ $firstEp == $FQDN_PATTERN ]]; then + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/kinghistorian.cfg + break + #else + # read -p "Please enter the correct FQDN:port: " firstEp + #fi + else + break + fi + done +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_data() { + ${csudo} mkdir -p ${data_dir} + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function clean_service_on_sysvinit() { + if pidof khserver &> /dev/null; then + ${csudo} service khserver stop || : + fi + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/khserver ]; then + ${csudo} chkconfig --del khserver || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/khserver ]; then + ${csudo} insserv -r khserver || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/khserver ]; then + ${csudo} update-rc.d -f khserver remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/khserver || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install khserver service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/khserver.deb ${install_main_dir}/init.d/khserver + ${csudo} cp ${script_dir}/init.d/khserver.deb ${service_config_dir}/khserver && ${csudo} chmod a+x ${service_config_dir}/khserver + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/khserver.rpm ${install_main_dir}/init.d/khserver + ${csudo} cp ${script_dir}/init.d/khserver.rpm ${service_config_dir}/khserver && ${csudo} chmod a+x ${service_config_dir}/khserver + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add khserver || : + ${csudo} chkconfig --level 2345 khserver on || : + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv khserver || : + ${csudo} insserv -d khserver || : + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d khserver defaults || : + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + khserver_service_config="${service_config_dir}/khserver.service" + if systemctl is-active --quiet khserver; then + echo "KingHistorian is running, stopping it..." + ${csudo} systemctl stop khserver &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable khserver &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${khserver_service_config} + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + if systemctl is-active --quiet nginxd; then + echo "Nginx for KingHistorian is running, stopping it..." + ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${nginx_service_config} + fi +} + +function install_service_on_systemd() { + clean_service_on_systemd + + service_config="${service_config_dir}/khserver.service" + ${csudo} bash -c "echo '[Unit]' >> ${service_config}" + ${csudo} bash -c "echo 'Description=KingHistorian server service' >> ${service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${service_config}" + ${csudo} bash -c "echo >> ${service_config}" + ${csudo} bash -c "echo '[Service]' >> ${service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/khserver' >> ${service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/kinghistorian/bin/startPre.sh' >> ${service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${service_config}" + ${csudo} bash -c "echo >> ${service_config}" + ${csudo} bash -c "echo '[Install]' >> ${service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${service_config}" + ${csudo} systemctl enable khserver + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=KingHistorian arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Description=Nginx For KingHistorian Service' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Type=forking' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${nginx_service_config}" + if ! ${csudo} systemctl enable nginxd &> /dev/null; then + ${csudo} systemctl daemon-reexec + ${csudo} systemctl enable nginxd + fi + ${csudo} systemctl start nginxd + fi +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop khserver + kill_process khserver + fi +} + +vercomp () { + if [[ $1 == $2 ]]; then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +function is_version_compatible() { + curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` + + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` + else + min_compatible_version=$(${script_dir}/bin/khserver -V | head -1 | cut -d ' ' -f 5) + fi + + vercomp $curr_version $min_compatible_version + case $? in + 0) return 0;; + 1) return 0;; + 2) return 1;; + esac +} + +function update() { + # Start to update + if [ ! -e kinghistorian.tar.gz ]; then + echo "File kinghistorian.tar.gz does not exist" + exit 1 + fi + tar -zxf kinghistorian.tar.gz + install_jemalloc + + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + + echo -e "${GREEN}Start to update KingHistorian...${NC}" + # Stop the service if running + if pidof khserver &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop khserver || : + elif ((${service_mod}==1)); then + ${csudo} service khserver stop || : + else + kill_process khserver + fi + sleep 1 + fi + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx + fi + sleep 1 + fi + fi + + install_main_path + + install_log + install_header + install_lib +# if [ "$pagMode" != "lite" ]; then +# install_connector +# fi +# install_examples + if [ -z $1 ]; then + install_bin + install_service + install_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if openresty is installed + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for KingHistorian is updated successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for KingHistorian does not work! Please try again!\033[0m" + fi + fi + fi + + #echo + #echo -e "\033[44;32;1mKingHistorian is updated successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure KingHistorian ${NC}: edit /etc/kinghistorian/kinghistorian.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start KingHistorian ${NC}: ${csudo} systemctl start khserver${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start KingHistorian ${NC}: ${csudo} service khserver start${NC}" + else + echo -e "${GREEN_DARK}To start KingHistorian ${NC}: ./khserver${NC}" + fi + + if [ ${openresty_work} = 'true' ]; then + echo -e "${GREEN_DARK}To access KingHistorian ${NC}: use ${GREEN_UNDERLINE}khclient -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + else + echo -e "${GREEN_DARK}To access KingHistorian ${NC}: use ${GREEN_UNDERLINE}khclient -h $serverFqdn${NC} in shell${NC}" + fi + + echo + echo -e "\033[44;32;1mKingHistorian is updated successfully!${NC}" + else + install_bin + install_config + + echo + echo -e "\033[44;32;1mKingHistorian client is updated successfully!${NC}" + fi + + rm -rf $(tar -tf kinghistorian.tar.gz) +} + +function install() { + # Start to install + if [ ! -e kinghistorian.tar.gz ]; then + echo "File kinghistorian.tar.gz does not exist" + exit 1 + fi + tar -zxf kinghistorian.tar.gz + + echo -e "${GREEN}Start to install KingHistorian...${NC}" + + install_main_path + + if [ -z $1 ]; then + install_data + fi + + install_log + install_header + install_lib + install_jemalloc +# if [ "$pagMode" != "lite" ]; then +# install_connector +# fi +# install_examples + + if [ -z $1 ]; then # install service and client + # For installing new + install_bin + install_service + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for KingHistorian is installed successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for KingHistorian does not work! Please try again!\033[0m" + fi + fi + fi + + install_config + + # Ask if to start the service + #echo + #echo -e "\033[44;32;1mKingHistorian is installed successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure KingHistorian ${NC}: edit /etc/kinghistorian/kinghistorian.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start KingHistorian ${NC}: ${csudo} systemctl start khserver${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start KingHistorian ${NC}: ${csudo} service khserver start${NC}" + else + echo -e "${GREEN_DARK}To start KingHistorian ${NC}: khserver${NC}" + fi + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access KingHistorian ${NC}: khclient -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access KingHistorian ${NC}: khclient -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access KingHistorian ${NC}: khclient -h $serverFqdn${GREEN_DARK} to login into KingHistorian server${NC}" + echo + fi + echo -e "\033[44;32;1mKingHistorian is installed successfully!${NC}" + echo + else # Only install client + install_bin + install_config + + echo + echo -e "\033[44;32;1mKingHistorian client is installed successfully!${NC}" + fi + + rm -rf $(tar -tf kinghistorian.tar.gz) +} + + +## ==============================Main program starts from here============================ +serverFqdn=$(hostname) +if [ "$verType" == "server" ]; then + # Install server and client + if [ -x ${bin_dir}/khserver ]; then + update_flag=1 + update + else + install + fi +elif [ "$verType" == "client" ]; then + interactiveFqdn=no + # Only install client + if [ -x ${bin_dir}/khclient ]; then + update_flag=1 + update client + else + install client + fi +else + echo "please input correct verType" +fi diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index 05eb09d8f3a8b5237c36714e964530b877e332de..1935372a6b6289ef8f5e762a0d3a4400f984c632 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -210,13 +210,6 @@ function install_lib() { ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi - #if [ "$verMode" == "cluster" ]; then - # # Compatible with version 1.5 - # ${csudo} mkdir -p ${v15_java_app_dir} - # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar - # ${csudo} chmod 777 ${v15_java_app_dir} || : - #fi - ${csudo} ldconfig } @@ -328,7 +321,7 @@ function set_hostname() { ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: fi - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/power.cfg serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -363,7 +356,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/power.cfg serverFqdn=$localFqdn echo return @@ -385,7 +378,7 @@ function set_ipAsFqdn() { read -p "Please choose an IP from local IP list:" localFqdn else # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/power.cfg serverFqdn=$localFqdn break fi @@ -432,16 +425,14 @@ function local_fqdn_check() { } function install_config() { - #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + if [ ! -f ${cfg_install_dir}/power.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/cfg/power.cfg ] && ${csudo} cp ${script_dir}/cfg/power.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi - ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} cp -f ${script_dir}/cfg/power.cfg ${install_main_dir}/cfg/power.cfg.org + ${csudo} ln -s ${cfg_install_dir}/power.cfg ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client @@ -471,7 +462,7 @@ function install_config() { # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/power.cfg break #else # read -p "Please enter the correct FQDN:port: " firstEp @@ -607,7 +598,7 @@ function clean_service_on_systemd() { if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" if systemctl is-active --quiet nginxd; then - echo "Nginx for TDengine is running, stopping it..." + echo "Nginx for PowerDB is running, stopping it..." ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null @@ -646,7 +637,7 @@ function install_service_on_systemd() { tarbitratord_service_config="${service_config_dir}/tarbitratord.service" ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=PowerDB arbitrator service' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo >> ${tarbitratord_service_config}" @@ -828,7 +819,7 @@ function update_PowerDB() { #echo #echo -e "\033[44;32;1mPowerDB is updated successfully!${NC}" echo - echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/power/taos.cfg" + echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/power/power.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start PowerDB ${NC}: ${csudo} systemctl start powerd${NC}" elif ((${service_mod}==1)); then @@ -905,7 +896,7 @@ function install_PowerDB() { #echo #echo -e "\033[44;32;1mPowerDB is installed successfully!${NC}" echo - echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/power/taos.cfg" + echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/power/power.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start PowerDB ${NC}: ${csudo} systemctl start powerd${NC}" elif ((${service_mod}==1)); then diff --git a/packaging/tools/install_pro.sh b/packaging/tools/install_pro.sh index 527f9a231e5a97fa086ef655cd420abc61677fcf..260e315a5e9745c5c60c41f19b086074b18f6c1b 100755 --- a/packaging/tools/install_pro.sh +++ b/packaging/tools/install_pro.sh @@ -316,7 +316,7 @@ function set_hostname() { ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: fi - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/prodb.cfg serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -351,7 +351,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/prodb.cfg serverFqdn=$localFqdn echo return @@ -373,7 +373,7 @@ function set_ipAsFqdn() { read -p "Please choose an IP from local IP list:" localFqdn else # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/prodb.cfg serverFqdn=$localFqdn break fi @@ -420,14 +420,14 @@ function local_fqdn_check() { } function install_config() { - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + if [ ! -f ${cfg_install_dir}/prodb.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/cfg/prodb.cfg ] && ${csudo} cp ${script_dir}/cfg/prodb.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi - ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} cp -f ${script_dir}/cfg/prodb.cfg ${install_main_dir}/cfg/prodb.cfg.org + ${csudo} ln -s ${cfg_install_dir}/prodb.cfg ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client @@ -457,7 +457,7 @@ function install_config() { # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/prodb.cfg break #else # read -p "Please enter the correct FQDN:port: " firstEp @@ -805,7 +805,7 @@ function update_prodb() { #echo #echo -e "\033[44;32;1mProDB is updated successfully!${NC}" echo - echo -e "${GREEN_DARK}To configure ProDB ${NC}: edit /etc/ProDB/taos.cfg" + echo -e "${GREEN_DARK}To configure ProDB ${NC}: edit /etc/ProDB/prodb.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start ProDB ${NC}: ${csudo} systemctl start prodbs${NC}" elif ((${service_mod}==1)); then @@ -882,7 +882,7 @@ function install_prodb() { #echo #echo -e "\033[44;32;1mProDB is installed successfully!${NC}" echo - echo -e "${GREEN_DARK}To configure ProDB ${NC}: edit /etc/ProDB/taos.cfg" + echo -e "${GREEN_DARK}To configure ProDB ${NC}: edit /etc/ProDB/prodb.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start ProDB ${NC}: ${csudo} systemctl start prodbs${NC}" elif ((${service_mod}==1)); then diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh index 52e08cb6b0d00b25686b87e2f066401e0388d4ce..c5acf9ed27aaa61b53f41c1e380a96a4c86fc2a2 100755 --- a/packaging/tools/install_tq.sh +++ b/packaging/tools/install_tq.sh @@ -210,13 +210,6 @@ function install_lib() { ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi - #if [ "$verMode" == "cluster" ]; then - # # Compatible with version 1.5 - # ${csudo} mkdir -p ${v15_java_app_dir} - # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar - # ${csudo} chmod 777 ${v15_java_app_dir} || : - #fi - ${csudo} ldconfig } @@ -328,7 +321,7 @@ function set_hostname() { ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: fi - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/tq.cfg serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -363,7 +356,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/tq.cfg serverFqdn=$localFqdn echo return @@ -385,7 +378,7 @@ function set_ipAsFqdn() { read -p "Please choose an IP from local IP list:" localFqdn else # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/tq.cfg serverFqdn=$localFqdn break fi @@ -432,16 +425,14 @@ function local_fqdn_check() { } function install_config() { - #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + if [ ! -f ${cfg_install_dir}/tq.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/cfg/tq.cfg ] && ${csudo} cp ${script_dir}/cfg/tq.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi - ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} cp -f ${script_dir}/cfg/tq.cfg ${install_main_dir}/cfg/tq.cfg.org + ${csudo} ln -s ${cfg_install_dir}/tq.cfg ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client @@ -471,7 +462,7 @@ function install_config() { # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/tq.cfg break #else # read -p "Please enter the correct FQDN:port: " firstEp @@ -607,7 +598,7 @@ function clean_service_on_systemd() { if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" if systemctl is-active --quiet nginxd; then - echo "Nginx for TDengine is running, stopping it..." + echo "Nginx for TQ is running, stopping it..." ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null @@ -646,7 +637,7 @@ function install_service_on_systemd() { tarbitratord_service_config="${service_config_dir}/tarbitratord.service" ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TQ arbitrator service' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo >> ${tarbitratord_service_config}" @@ -828,7 +819,7 @@ function update_tq() { #echo #echo -e "\033[44;32;1mTQ is updated successfully!${NC}" echo - echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/taos.cfg" + echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/tq.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} systemctl start tqd${NC}" elif ((${service_mod}==1)); then @@ -905,7 +896,7 @@ function install_tq() { #echo #echo -e "\033[44;32;1mTQ is installed successfully!${NC}" echo - echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/taos.cfg" + echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/tq.cfg" if ((${service_mod}==0)); then echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} systemctl start tqd${NC}" elif ((${service_mod}==1)); then diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index eb61963de397db7773bf713bcc18de9f19b794b5..ea5ab8b1ba6b75bf34001a3e1547a016567d002d 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -282,8 +282,6 @@ function install_header() { } function install_config() { - #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} [ -f ${script_dir}/../cfg/taos.cfg ] && @@ -316,11 +314,6 @@ function install_data() { } function install_connector() { - if [ -d "${source_dir}/src/connector/grafanaplugin/dist" ]; then - ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" - fi if find ${source_dir}/src/connector/go -mindepth 1 -maxdepth 1 | read; then ${csudo} cp -r ${source_dir}/src/connector/go ${install_main_dir}/connector else @@ -330,8 +323,9 @@ function install_connector() { ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null else - ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_2_dir}/connector} - ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null || cp ${binary_dir}/build/lib/*.jar ${install_main_2_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || ${csudo} chmod 777 ${install_main_2_dir}/connector/*.jar || echo &> /dev/null + ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_2_dir}/connector + ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null + ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_2_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_2_dir}/connector/*.jar || echo &> /dev/null fi } diff --git a/packaging/tools/makearbi_jh.sh b/packaging/tools/makearbi_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..5457b163599421d0a5917156efde1c8814a6f514 --- /dev/null +++ b/packaging/tools/makearbi_jh.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/jh_iot-enterprise-arbitrator-${version}" +else + install_dir="${release_dir}/jh_iot-arbitrator-${version}" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi_jh.sh" +install_files="${script_dir}/install_arbi_jh.sh" + +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi_jh.sh || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makearbi_kh.sh b/packaging/tools/makearbi_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..c7fa40eb4f1fc4003e6a584bdc5c4534616754d6 --- /dev/null +++ b/packaging/tools/makearbi_kh.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/KingHistorian-enterprise-arbitrator-${version}" +else + install_dir="${release_dir}/KingHistorian-arbitrator-${version}" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi_kh.sh" +install_files="${script_dir}/install_arbi_kh.sh" + +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi_kh.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makearbi_power.sh b/packaging/tools/makearbi_power.sh index fd50ecd43878de08e7bb94249da8cb64c3630e6e..f4d14809f70225406adae49f30f1dcbe4818f014 100755 --- a/packaging/tools/makearbi_power.sh +++ b/packaging/tools/makearbi_power.sh @@ -34,7 +34,6 @@ fi bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi_power.sh" install_files="${script_dir}/install_arbi_power.sh" -#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord diff --git a/packaging/tools/makearbi_pro.sh b/packaging/tools/makearbi_pro.sh index 6ce3765e44acc408ced9730c54b793338eb37b38..5b5cd5447c2630f0e6bd9ba53d50856761049bc6 100755 --- a/packaging/tools/makearbi_pro.sh +++ b/packaging/tools/makearbi_pro.sh @@ -34,7 +34,6 @@ fi bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi_pro.sh" install_files="${script_dir}/install_arbi_pro.sh" -#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord diff --git a/packaging/tools/makearbi_tq.sh b/packaging/tools/makearbi_tq.sh index c10dfec255d411965a3887942e5d2aded4635979..fb5b42a062b1bc34f4104e314de2adeca462d8b9 100755 --- a/packaging/tools/makearbi_tq.sh +++ b/packaging/tools/makearbi_tq.sh @@ -34,7 +34,6 @@ fi bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi_tq.sh" install_files="${script_dir}/install_arbi_tq.sh" -#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 8fc431bfbc66d4f9d482ab5885d282081139ef4d..2275d09f8d8622b137e9c28b31f39f67d6ccd49b 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -46,7 +46,8 @@ if [ "$osType" != "Darwin" ]; then bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \ - ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb" + ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + #${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else @@ -150,11 +151,6 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - if [ -d "${connector_dir}/grafanaplugin/dist" ]; then - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" - fi if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then cp -r ${connector_dir}/go ${install_dir}/connector else diff --git a/packaging/tools/makeclient_jh.sh b/packaging/tools/makeclient_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..bfbdcfc578bc7f8dfb15fef302d9817014ff3bef --- /dev/null +++ b/packaging/tools/makeclient_jh.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# +# Generate tar.gz package for linux client in all os system +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +if [ "$osType" != "Darwin" ]; then + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/../..)" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/../.. +fi + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' + +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/jh_iot-enterprise-client-${version}" +else + install_dir="${release_dir}/jh_iot-client-${version}" +fi + +# Directories and files. + +if [ "$osType" != "Darwin" ]; then + lib_files="${build_dir}/lib/libtaos.so.${version}" +else + bin_files="${build_dir}/bin/taos ${script_dir}/remove_client_jh.sh" + lib_files="${build_dir}/lib/libtaos.${version}.dylib" +fi + +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install_client_jh.sh" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg + +sed -i '/dataDir/ {s/taos/jh_taos/g}' ${install_dir}/cfg/taos.cfg +sed -i '/logDir/ {s/taos/jh_taos/g}' ${install_dir}/cfg/taos.cfg +sed -i "s/TDengine/jh_iot/g" ${install_dir}/cfg/taos.cfg + +mkdir -p ${install_dir}/bin +if [ "$osType" != "Darwin" ]; then + if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taos + cp ${build_dir}/bin/taos ${install_dir}/bin/jh_taos + cp ${script_dir}/remove_client_jh.sh ${install_dir}/bin + else + cp ${build_dir}/bin/taos ${install_dir}/bin/jh_taos + cp ${script_dir}/remove_client_jh.sh ${install_dir}/bin + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/jhdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/jh_taosdump + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + #cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin + fi +else + cp ${bin_files} ${install_dir}/bin +fi +chmod a+x ${install_dir}/bin/* || : + +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + +cd ${install_dir} + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f jh_taos.tar.gz * --remove-files || : +else + tar -zcv -f jh_taos.tar.gz * || : + mv jh_taos.tar.gz .. + rm -rf ./* + mv ../jh_taos.tar.gz . +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$osType" == "Darwin" ]; then + sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client_jh.sh >> install_client_jh_temp.sh + mv install_client_jh_temp.sh ${install_dir}/install_client_jh.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client_jh.sh >> install_client_jh_temp.sh + mv install_client_jh_temp.sh ${install_dir}/install_client_jh.sh +fi +chmod a+x ${install_dir}/install_client_jh.sh + +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stable or beta" + exit 1 +fi + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +else + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || : + mv "$(basename ${pkg_name}).tar.gz" .. + rm -rf ./* + mv ../"$(basename ${pkg_name}).tar.gz" . +fi + +cd ${curr_dir} diff --git a/packaging/tools/makeclient_kh.sh b/packaging/tools/makeclient_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..015de87e0f26d738490e8045ba21a08511d9a62b --- /dev/null +++ b/packaging/tools/makeclient_kh.sh @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Generate tar.gz package for linux client in all os system +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +if [ "$osType" != "Darwin" ]; then + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/../..)" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/../.. +fi + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' + +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/KingHistorian-enterprise-client-${version}" +else + install_dir="${release_dir}/KingHistorian-client-${version}" +fi + +# Directories and files. + +if [ "$osType" != "Darwin" ]; then + lib_files="${build_dir}/lib/libtaos.so.${version}" +else + bin_files="${build_dir}/bin/taos ${script_dir}/remove_client_kh.sh" + lib_files="${build_dir}/lib/libtaos.${version}.dylib" +fi + +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install_client_kh.sh" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/kinghistorian.cfg + +sed -i '/dataDir/ {s/taos/kinghistorian/g}' ${install_dir}/cfg/kinghistorian.cfg +sed -i '/logDir/ {s/taos/kinghistorian/g}' ${install_dir}/cfg/kinghistorian.cfg +sed -i "s/TDengine/KingHistorian/g" ${install_dir}/cfg/kinghistorian.cfg +sed -i "s/TAOS/KingHistorian/g" ${install_dir}/cfg/kinghistorian.cfg + +mkdir -p ${install_dir}/bin +if [ "$osType" != "Darwin" ]; then + if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taos + cp ${build_dir}/bin/taos ${install_dir}/bin/khclient + cp ${script_dir}/remove_client_kh.sh ${install_dir}/bin + else + cp ${build_dir}/bin/taos ${install_dir}/bin/khclient + cp ${script_dir}/remove_client_kh.sh ${install_dir}/bin + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/khdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/khdump + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + #cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin + fi +else + cp ${bin_files} ${install_dir}/bin +fi +chmod a+x ${install_dir}/bin/* || : + +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + +cd ${install_dir} + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f kinghistorian.tar.gz * --remove-files || : +else + tar -zcv -f kinghistorian.tar.gz * || : + mv kinghistorian.tar.gz .. + rm -rf ./* + mv ../kinghistorian.tar.gz . +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$osType" == "Darwin" ]; then + sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client_kh.sh >> install_client_kh_temp.sh + mv install_client_kh_temp.sh ${install_dir}/install_client_kh.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client_kh.sh >> install_client_kh_temp.sh + mv install_client_kh_temp.sh ${install_dir}/install_client_kh.sh +fi +chmod a+x ${install_dir}/install_client_kh.sh + +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stable or beta" + exit 1 +fi + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +else + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || : + mv "$(basename ${pkg_name}).tar.gz" .. + rm -rf ./* + mv ../"$(basename ${pkg_name}).tar.gz" . +fi + +cd ${curr_dir} diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 89591cac234b190f55d144ccf98cb2d5c70a7936..a0a0070a97be4938b64b17a34ab7ec49d606edc6 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -109,15 +109,15 @@ if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos cp ${build_dir}/bin/taos ${install_dir}/bin/power - cp ${script_dir}/remove_power.sh ${install_dir}/bin + cp ${script_dir}/remove_client_power.sh ${install_dir}/bin else cp ${build_dir}/bin/taos ${install_dir}/bin/power - cp ${script_dir}/remove_power.sh ${install_dir}/bin + cp ${script_dir}/remove_client_power.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin - cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin + #cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin fi else cp ${bin_files} ${install_dir}/bin @@ -210,11 +210,6 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - if [ -d "${connector_dir}/grafanaplugin/dist" ]; then - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" - fi if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then cp -r ${connector_dir}/go ${install_dir}/connector else diff --git a/packaging/tools/makeclient_pro.sh b/packaging/tools/makeclient_pro.sh index 599c91fbf082955887c677b750aa12f946c0890b..31de91263dd33cd930a0a30784a226f68a98768d 100755 --- a/packaging/tools/makeclient_pro.sh +++ b/packaging/tools/makeclient_pro.sh @@ -58,26 +58,26 @@ install_files="${script_dir}/install_client_pro.sh" # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc -mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/prodb.cfg -sed -i '/dataDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/taos.cfg -sed -i '/logDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/taos.cfg -sed -i "s/TDengine/ProDB/g" ${install_dir}/cfg/taos.cfg +sed -i '/dataDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/prodb.cfg +sed -i '/logDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/prodb.cfg +sed -i "s/TDengine/ProDB/g" ${install_dir}/cfg/prodb.cfg mkdir -p ${install_dir}/bin if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos cp ${build_dir}/bin/taos ${install_dir}/bin/prodbc - cp ${script_dir}/remove_pro.sh ${install_dir}/bin + cp ${script_dir}/remove_client_pro.sh ${install_dir}/bin else cp ${build_dir}/bin/taos ${install_dir}/bin/prodbc - cp ${script_dir}/remove_pro.sh ${install_dir}/bin + cp ${script_dir}/remove_client_pro.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/prodemo cp ${build_dir}/bin/taosdump ${install_dir}/bin/prodump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin - cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin + #cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin fi else cp ${bin_files} ${install_dir}/bin @@ -172,11 +172,6 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - if [ -d "${connector_dir}/grafanaplugin/dist" ]; then - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" - fi if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then cp -r ${connector_dir}/go ${install_dir}/connector else diff --git a/packaging/tools/makeclient_tq.sh b/packaging/tools/makeclient_tq.sh index 03d9b13059daadfdc7207c78b6f89cae321f25ac..52293ff47e15bf888f7e9963e1f02aacd718a607 100755 --- a/packaging/tools/makeclient_tq.sh +++ b/packaging/tools/makeclient_tq.sh @@ -40,13 +40,6 @@ fi # Directories and files. if [ "$osType" != "Darwin" ]; then -# if [ "$pagMode" == "lite" ]; then -# strip ${build_dir}/bin/tqd -# strip ${build_dir}/bin/tq -# bin_files="${build_dir}/bin/tq ${script_dir}/remove_client_tq.sh" -# else -# bin_files="${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${script_dir}/remove_client_tq.sh ${script_dir}/set_core.sh" -# fi lib_files="${build_dir}/lib/libtaos.so.${version}" else bin_files="${build_dir}/bin/tq ${script_dir}/remove_client_tq.sh" @@ -65,26 +58,26 @@ install_files="${script_dir}/install_client_tq.sh" # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc -mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/tq.cfg -sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg -sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg -sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/taos.cfg +sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/tq.cfg +sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/tq.cfg +sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/tq.cfg mkdir -p ${install_dir}/bin if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos cp ${build_dir}/bin/taos ${install_dir}/bin/tq - cp ${script_dir}/remove_tq.sh ${install_dir}/bin + cp ${script_dir}/remove_client_tq.sh ${install_dir}/bin else cp ${build_dir}/bin/taos ${install_dir}/bin/tq - cp ${script_dir}/remove_tq.sh ${install_dir}/bin + cp ${script_dir}/remove_client_tq.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/tqdemo cp ${build_dir}/bin/taosdump ${install_dir}/bin/tqdump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin - cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin + #cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin fi else cp ${bin_files} ${install_dir}/bin @@ -177,11 +170,6 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi - if [ -d "${connector_dir}/grafanaplugin/dist" ]; then - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" - fi if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then cp -r ${connector_dir}/go ${install_dir}/connector else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e9266ec80da293571ece07dab9c724b5b8c12adf..1b11973e3c232f3b00cd0f1d1db8c903e8046da6 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -63,6 +63,8 @@ init_file_deb=${script_dir}/../deb/taosd init_file_rpm=${script_dir}/../rpm/taosd init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord +init_file_nginx_deb=${script_dir}/../deb/nginxd +init_file_nginx_rpm=${script_dir}/../rpm/nginxd # make directories. mkdir -p ${install_dir} @@ -73,6 +75,8 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/taos mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/taosd.rpm mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : +mkdir -p ${install_dir}/init.d && cp ${init_file_nginx_deb} ${install_dir}/init.d/nginxd.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_nginx_rpm} ${install_dir}/init.d/nginxd.rpm || : if [ -f ${build_dir}/bin/jemalloc-config ]; then mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} @@ -183,11 +187,6 @@ connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - if [ -d "${connector_dir}/grafanaplugin/dist" ]; then - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bundled dir not found, please check if you want to use it!" - fi if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then cp -r ${connector_dir}/go ${install_dir}/connector else diff --git a/packaging/tools/makepkg_jh.sh b/packaging/tools/makepkg_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..d7b7746849578b0e2774ddf442566196102ea561 --- /dev/null +++ b/packaging/tools/makepkg_jh.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# +# Generate tar.gz package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 +versionComp=$9 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +# package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/jh_iot-enterprise-server-${version}" +else + install_dir="${release_dir}/jh_iot-server-${version}" +fi + +lib_files="${build_dir}/lib/libtaos.so.${version}" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi +install_files="${script_dir}/install_jh.sh" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg +mkdir -p ${install_dir}/bin + +# bin +if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taosd + strip ${build_dir}/bin/taos +else + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/jhdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/jh_taosdump + cp ${build_dir}/bin/tarbitrator ${install_dir}/bin + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/startPre.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin +fi +cp ${build_dir}/bin/taos ${install_dir}/bin/jh_taos +cp ${build_dir}/bin/taosd ${install_dir}/bin/jh_taosd +cp ${script_dir}/remove_jh.sh ${install_dir}/bin +chmod a+x ${install_dir}/bin/* || : + +# cluster +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_jh.sh >> remove_jh_temp.sh + mv remove_jh_temp.sh ${install_dir}/bin/remove_jh.sh + + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd + cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png + rm -rf ${install_dir}/nginxd/png + + # replace the OEM name + sed -i -e 's/www.taosdata.com/www.jhict.com/g' $(grep -r 'www.taosdata.com' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/2017/2021/g' $(grep -r '2017' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/TAOS Data/Jinheng Technology/g' $(grep -r 'TAOS Data' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/taosd/jh_taosd/g' `grep -r 'taosd' ${install_dir}/nginxd | grep -E '*\.js\s*.*' | sed -r -e 's/(.*\.js):\s*(.*)/\1/g' | sort | uniq` + sed -i -e 's/taosd<\/th>/jh_taosd<\/th>/g' ${install_dir}/nginxd/admin/monitor.html + sed -i -e "s/data:\['taosd', 'system'\],/data:\['jh_taosd', 'system'\],/g" ${install_dir}/nginxd/admin/monitor.html + sed -i -e "s/name: 'taosd',/name: 'jh_taosd',/g" ${install_dir}/nginxd/admin/monitor.html + sed -i "s/TDengine/jh_iot/g" ${install_dir}/nginxd/admin/*.html + sed -i "s/TDengine/jh_iot/g" ${install_dir}/nginxd/admin/js/*.js + + if [ "$cpuType" == "aarch64" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ + elif [ "$cpuType" == "aarch32" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/ + fi + rm -rf ${install_dir}/nginxd/sbin/arm +fi + +sed -i '/dataDir/ {s/taos/jh_taos/g}' ${install_dir}/cfg/taos.cfg +sed -i '/logDir/ {s/taos/jh_taos/g}' ${install_dir}/cfg/taos.cfg +sed -i "s/TDengine/jh_iot/g" ${install_dir}/cfg/taos.cfg +sed -i "s/support@taosdata.com/jhkj@njsteel.com.cn/g" ${install_dir}/cfg/taos.cfg +sed -i "s/taos client/client/g" ${install_dir}/cfg/taos.cfg +sed -i "s/taosd/server/g" ${install_dir}/cfg/taos.cfg + +cd ${install_dir} +tar -zcv -f jh_taos.tar.gz * --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar jh_taos.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_jh.sh >> install_jh_temp.sh + mv install_jh_temp.sh ${install_dir}/install_jh.sh +fi +if [ "$pagMode" == "lite" ]; then + sed -e "s/pagMode=full/pagMode=lite/g" -e "s/taos_history/jh_taos_history/g" ${install_dir}/install.sh >> install_jh_temp.sh + mv install_jh_temp.sh ${install_dir}/install_jh.sh +fi + +sed -i "/install_connector$/d" ${install_dir}/install_jh.sh +sed -i "/install_examples$/d" ${install_dir}/install_jh.sh +chmod a+x ${install_dir}/install_jh.sh + +# Copy driver +mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makepkg_kh.sh b/packaging/tools/makepkg_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..16b5b1a50817aadc1d078814f5db9137aeaa4415 --- /dev/null +++ b/packaging/tools/makepkg_kh.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# +# Generate tar.gz package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 +versionComp=$9 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +# package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/KingHistorian-enterprise-server-${version}" +else + install_dir="${release_dir}/KingHistorian-server-${version}" +fi + +lib_files="${build_dir}/lib/libtaos.so.${version}" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi +install_files="${script_dir}/install_kh.sh" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/kinghistorian.cfg +mkdir -p ${install_dir}/bin + +# bin +if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taosd + strip ${build_dir}/bin/taos +else + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/khdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/khdump + cp ${build_dir}/bin/tarbitrator ${install_dir}/bin + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/startPre.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin +fi +cp ${build_dir}/bin/taos ${install_dir}/bin/khclient +cp ${build_dir}/bin/taosd ${install_dir}/bin/khserver +cp ${script_dir}/remove_kh.sh ${install_dir}/bin +chmod a+x ${install_dir}/bin/* || : + +# cluster +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_kh.sh >> remove_kh_temp.sh + mv remove_kh_temp.sh ${install_dir}/bin/remove_kh.sh + + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd + cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png + rm -rf ${install_dir}/nginxd/png + + sed -i -e 's/www.taosdata.com/www.wellintech.com/g' $(grep -r 'www.taosdata.com' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/2017/2021/g' $(grep -r '2017' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/TAOS Data/Wellintech/g' $(grep -r 'TAOS Data' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/taosd/khserver/g' `grep -r 'taosd' ${install_dir}/nginxd | grep -E '*\.js\s*.*' | sed -r -e 's/(.*\.js):\s*(.*)/\1/g' | sort | uniq` + sed -i -e 's/taosd<\/th>/khserver<\/th>/g' ${install_dir}/nginxd/admin/monitor.html + sed -i -e "s/data:\['taosd', 'system'\],/data:\['khserver', 'system'\],/g" ${install_dir}/nginxd/admin/monitor.html + sed -i -e "s/name: 'taosd',/name: 'khserver',/g" ${install_dir}/nginxd/admin/monitor.html + sed -i "s/TDengine/KingHistorian/g" ${install_dir}/nginxd/admin/*.html + sed -i "s/TDengine/KingHistorian/g" ${install_dir}/nginxd/admin/js/*.js + + if [ "$cpuType" == "aarch64" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ + elif [ "$cpuType" == "aarch32" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/ + fi + rm -rf ${install_dir}/nginxd/sbin/arm +fi + +sed -i '/dataDir/ {s/taos/kinghistorian/g}' ${install_dir}/cfg/kinghistorian.cfg +sed -i '/logDir/ {s/taos/kinghistorian/g}' ${install_dir}/cfg/kinghistorian.cfg +sed -i "s/TDengine/KingHistorian/g" ${install_dir}/cfg/kinghistorian.cfg +sed -i "s/support@taosdata.com/support@wellintech.com/g" ${install_dir}/cfg/kinghistorian.cfg +sed -i "s/taos client/khclient/g" ${install_dir}/cfg/kinghistorian.cfg +sed -i "s/taosd/khserver/g" ${install_dir}/cfg/kinghistorian.cfg +sed -i "s/TAOS/KingHistorian/g" ${install_dir}/cfg/kinghistorian.cfg + +cd ${install_dir} +tar -zcv -f kinghistorian.tar.gz * --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar kinghistorian.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_kh.sh >> install_kh_temp.sh + mv install_kh_temp.sh ${install_dir}/install_kh.sh +fi +if [ "$pagMode" == "lite" ]; then + sed -e "s/pagMode=full/pagMode=lite/g" -e "s/taos_history/kh_history/g" ${install_dir}/install.sh >> install_kh_temp.sh + mv install_kh_temp.sh ${install_dir}/install_kh.sh +fi + +sed -i "/install_connector$/d" ${install_dir}/install_kh.sh +sed -i "/install_examples$/d" ${install_dir}/install_kh.sh +chmod a+x ${install_dir}/install_kh.sh + +# Copy driver +mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index a2643b7486195041466d28d84d25a6b5aa05974e..e5c3c91ebf298e156c331ff2d1d37240b726d460 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -31,16 +31,6 @@ else install_dir="${release_dir}/PowerDB-server-${version}" fi -# Directories and files. -#if [ "$pagMode" == "lite" ]; then -# strip ${build_dir}/bin/taosd -# strip ${build_dir}/bin/taos -# bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${script_dir}/remove_power.sh" -#else -# bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh\ -# ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" -#fi - lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" if [ "$verMode" == "cluster" ]; then @@ -51,13 +41,6 @@ fi install_files="${script_dir}/install_power.sh" nginx_dir="${code_dir}/../../enterprise/src/plugins/web" -# Init file -#init_dir=${script_dir}/deb -#if [ $package_type = "centos" ]; then -# init_dir=${script_dir}/rpm -#fi -#init_files=${init_dir}/powerd -# temp use rpm's powerd. TODO: later modify according to os type init_file_deb=${script_dir}/../deb/powerd init_file_rpm=${script_dir}/../rpm/powerd init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord @@ -66,7 +49,7 @@ init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc -mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/power.cfg #mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/bin @@ -108,9 +91,9 @@ if [ "$verMode" == "cluster" ]; then sed -i "s/TDengine/PowerDB/g" ${install_dir}/nginxd/admin/*.html sed -i "s/TDengine/PowerDB/g" ${install_dir}/nginxd/admin/js/*.js - sed -i '/dataDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg - sed -i '/logDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg - sed -i "s/TDengine/PowerDB/g" ${install_dir}/cfg/taos.cfg + sed -i '/dataDir/ {s/taos/power/g}' ${install_dir}/cfg/power.cfg + sed -i '/logDir/ {s/taos/power/g}' ${install_dir}/cfg/power.cfg + sed -i "s/TDengine/PowerDB/g" ${install_dir}/cfg/power.cfg if [ "$cpuType" == "aarch64" ]; then cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ @@ -167,11 +150,6 @@ mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - if [ -d "${connector_dir}/grafanaplugin/dist" ]; then - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" - fi if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then cp -r ${connector_dir}/go ${install_dir}/connector else diff --git a/packaging/tools/makepkg_pro.sh b/packaging/tools/makepkg_pro.sh index e59cd65425ff2c9e9d37ac6fd2e548b2588dc76f..a03b306810caf2368f94721a90efb9bafa57d665 100755 --- a/packaging/tools/makepkg_pro.sh +++ b/packaging/tools/makepkg_pro.sh @@ -44,7 +44,7 @@ nginx_dir="${code_dir}/../../enterprise/src/plugins/web" # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc -mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/prodb.cfg mkdir -p ${install_dir}/bin # bin @@ -93,12 +93,12 @@ if [ "$verMode" == "cluster" ]; then rm -rf ${install_dir}/nginxd/sbin/arm fi -sed -i '/dataDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/taos.cfg -sed -i '/logDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/taos.cfg -sed -i "s/TDengine/ProDB/g" ${install_dir}/cfg/taos.cfg -sed -i "s/support@taosdata.com/support@hanatech.com.cn/g" ${install_dir}/cfg/taos.cfg -sed -i "s/taos client/prodbc/g" ${install_dir}/cfg/taos.cfg -sed -i "s/taosd/prodbs/g" ${install_dir}/cfg/taos.cfg +sed -i '/dataDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/prodb.cfg +sed -i '/logDir/ {s/taos/ProDB/g}' ${install_dir}/cfg/prodb.cfg +sed -i "s/TDengine/ProDB/g" ${install_dir}/cfg/prodb.cfg +sed -i "s/support@taosdata.com/support@hanatech.com.cn/g" ${install_dir}/cfg/prodb.cfg +sed -i "s/taos client/prodbc/g" ${install_dir}/cfg/prodb.cfg +sed -i "s/taosd/prodbs/g" ${install_dir}/cfg/prodb.cfg cd ${install_dir} tar -zcv -f prodb.tar.gz * --remove-files || : @@ -123,55 +123,9 @@ sed -i "/install_connector$/d" ${install_dir}/install_pro.sh sed -i "/install_examples$/d" ${install_dir}/install_pro.sh chmod a+x ${install_dir}/install_pro.sh -# Copy example code -#mkdir -p ${install_dir}/examples -#examples_dir="${top_dir}/tests/examples" -#cp -r ${examples_dir}/c ${install_dir}/examples -#sed -i '/passwd/ {s/taosdata/prodb/g}' ${install_dir}/examples/c/*.c -#sed -i '/root/ {s/taosdata/prodb/g}' ${install_dir}/examples/c/*.c -# -#if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then -# cp -r ${examples_dir}/JDBC ${install_dir}/examples -# cp -r ${examples_dir}/matlab ${install_dir}/examples -# mv ${install_dir}/examples/matlab/TDengineDemo.m ${install_dir}/examples/matlab/ProDBDemo.m -# sed -i '/password/ {s/taosdata/prodb/g}' ${install_dir}/examples/matlab/ProDBDemo.m -# cp -r ${examples_dir}/python ${install_dir}/examples -# sed -i '/password/ {s/taosdata/prodb/g}' ${install_dir}/examples/python/read_example.py -# cp -r ${examples_dir}/R ${install_dir}/examples -# sed -i '/password/ {s/taosdata/prodb/g}' ${install_dir}/examples/R/command.txt -# cp -r ${examples_dir}/go ${install_dir}/examples -# mv ${install_dir}/examples/go/taosdemo.go ${install_dir}/examples/go/prodemo.go -# sed -i '/root/ {s/taosdata/prodb/g}' ${install_dir}/examples/go/prodemo.go -#fi - # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt -# Copy connector -#connector_dir="${code_dir}/connector" -#mkdir -p ${install_dir}/connector -#if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then -# cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - -# if [ -d "${connector_dir}/grafanaplugin/dist" ]; then -# cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin -# else -# echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" -# fi -# if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then -# cp -r ${connector_dir}/go ${install_dir}/connector -# else -# echo "WARNING: go connector not found, please check if want to use it!" -# fi -# cp -r ${connector_dir}/python ${install_dir}/connector/ -# mv ${install_dir}/connector/python/taos ${install_dir}/connector/python/prodb -# sed -i '/password/ {s/taosdata/prodb/g}' ${install_dir}/connector/python/prodb/cinterface.py - -# sed -i '/password/ {s/taosdata/prodb/g}' ${install_dir}/connector/python/prodb/subscription.py - -# sed -i '/self._password/ {s/taosdata/prodb/g}' ${install_dir}/connector/python/prodb/connection.py -#fi - cd ${release_dir} if [ "$verMode" == "cluster" ]; then diff --git a/packaging/tools/makepkg_tq.sh b/packaging/tools/makepkg_tq.sh index 6f897de0ce5e7287e06719562199e8ed139b02ec..8abdfc2f7237ef465eac79907ad2b2a4c35c0463 100755 --- a/packaging/tools/makepkg_tq.sh +++ b/packaging/tools/makepkg_tq.sh @@ -31,16 +31,6 @@ else install_dir="${release_dir}/TQ-server-${version}" fi -# Directories and files. -#if [ "$pagMode" == "lite" ]; then -# strip ${build_dir}/bin/taosd -# strip ${build_dir}/bin/taos -# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" -#else -# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh\ -# ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" -#fi - lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" if [ "$verMode" == "cluster" ]; then @@ -51,34 +41,19 @@ fi install_files="${script_dir}/install_tq.sh" nginx_dir="${code_dir}/../../enterprise/src/plugins/web" -# Init file -#init_dir=${script_dir}/deb -#if [ $package_type = "centos" ]; then -# init_dir=${script_dir}/rpm -#fi -#init_files=${init_dir}/tqd -# temp use rpm's tqd. TODO: later modify according to os type -#init_file_deb=${script_dir}/../deb/tqd -#init_file_rpm=${script_dir}/../rpm/tqd -#init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord -#init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord - # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc -mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/tq.cfg -#mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/bin if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos -# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" cp ${build_dir}/bin/taos ${install_dir}/bin/tq cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd cp ${script_dir}/remove_tq.sh ${install_dir}/bin else -# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh ${script_dir}/set_core.sh" cp ${build_dir}/bin/taos ${install_dir}/bin/tq cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd cp ${script_dir}/remove_tq.sh ${install_dir}/bin @@ -92,11 +67,6 @@ else fi chmod a+x ${install_dir}/bin/* || : -#mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/tqd.deb -#mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/tqd.rpm -#mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : -#mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : - if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_tq.sh >> remove_tq_temp.sh mv remove_tq_temp.sh ${install_dir}/bin/remove_tq.sh @@ -108,9 +78,9 @@ if [ "$verMode" == "cluster" ]; then sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/*.html sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/js/*.js - sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg - sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg - sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/taos.cfg + sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/tq.cfg + sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/tq.cfg + sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/tq.cfg if [ "$cpuType" == "aarch64" ]; then cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ @@ -167,11 +137,6 @@ mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: - if [ -d "${connector_dir}/grafanaplugin/dist" ]; then - cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin - else - echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" - fi if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then cp -r ${connector_dir}/go ${install_dir}/connector else @@ -185,10 +150,6 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then sed -i '/self._password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/connection.py fi -# Copy release note -# cp ${script_dir}/release_note ${install_dir} - -# exit 1 cd ${release_dir} diff --git a/packaging/tools/remove_arbi_jh.sh b/packaging/tools/remove_arbi_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..8b690771c761ac51772dac83cafec46360a16be3 --- /dev/null +++ b/packaging/tools/remove_arbi_jh.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Script to stop the service and uninstall jh_iot's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "jh_iot's tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "jh_iot's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}jh_iot's arbitrator is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_arbi_kh.sh b/packaging/tools/remove_arbi_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..ec3254b01649add57f9485c59878059e086b2669 --- /dev/null +++ b/packaging/tools/remove_arbi_kh.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Script to stop the service and uninstall KingHistorian's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "KingHistorian's tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "KingHistorian's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}KingHistorian's arbitrator is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_arbi_power.sh b/packaging/tools/remove_arbi_power.sh index 077b19ec7d4208c604c2042c2aa1eacab2033c5b..f34cb4ffed63ae4db33080080a24101f44f3389d 100755 --- a/packaging/tools/remove_arbi_power.sh +++ b/packaging/tools/remove_arbi_power.sh @@ -127,4 +127,4 @@ clean_log ${csudo} rm -rf ${install_main_dir} echo -e "${GREEN}PowerDB's arbitrator is removed successfully!${NC}" -echo \ No newline at end of file +echo diff --git a/packaging/tools/remove_arbi_tq.sh b/packaging/tools/remove_arbi_tq.sh index 3d99b6d41a74938d74383df3d8cdfc75c2ebb7c8..27c78b28601a4c57482f9fc56a76175baf668948 100755 --- a/packaging/tools/remove_arbi_tq.sh +++ b/packaging/tools/remove_arbi_tq.sh @@ -127,4 +127,4 @@ clean_log ${csudo} rm -rf ${install_main_dir} echo -e "${GREEN}TQ's arbitrator is removed successfully!${NC}" -echo \ No newline at end of file +echo diff --git a/packaging/tools/remove_client_jh.sh b/packaging/tools/remove_client_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..a3f5dfd10debb0a28211b3682becd083d49ca9c6 --- /dev/null +++ b/packaging/tools/remove_client_jh.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Script to stop the client and uninstall database, but retain the config and log files. +set -e +# set -x + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/jh_taos" + +log_link_dir="/usr/local/jh_taos/log" +cfg_link_dir="/usr/local/jh_taos/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +function kill_client() { + if [ -n "$(pidof jh_taos)" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/jh_taos || : + ${csudo} rm -f ${bin_link_dir}/jhdemo || : + ${csudo} rm -f ${bin_link_dir}/jh_taosdump || : + ${csudo} rm -f ${bin_link_dir}/rmjh || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +# Stop client. +kill_client +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}jh_iot client is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_client_kh.sh b/packaging/tools/remove_client_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..6a44e875e3426b14400508b1bdbd7510c2ae49cb --- /dev/null +++ b/packaging/tools/remove_client_kh.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Script to stop the client and uninstall database, but retain the config and log files. +set -e +# set -x + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/kinghistorian" + +log_link_dir="/usr/local/kinghistorian/log" +cfg_link_dir="/usr/local/kinghistorian/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +function kill_client() { + if [ -n "$(pidof khclient)" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/khclient || : + ${csudo} rm -f ${bin_link_dir}/khdemo || : + ${csudo} rm -f ${bin_link_dir}/khdump || : + ${csudo} rm -f ${bin_link_dir}/rmkh || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +# Stop client. +kill_client +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}KingHistorian client is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_jh.sh b/packaging/tools/remove_jh.sh new file mode 100755 index 0000000000000000000000000000000000000000..b962b824fdeeda632be28eeeaa97199adcd6ca2c --- /dev/null +++ b/packaging/tools/remove_jh.sh @@ -0,0 +1,209 @@ +#!/bin/bash +# +# Script to stop the service and uninstall jh_taos, but retain the config, data and log files. + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/jh_taos" +data_link_dir="/usr/local/jh_taos/data" +log_link_dir="/usr/local/jh_taos/log" +cfg_link_dir="/usr/local/jh_taos/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" +install_nginxd_dir="/usr/local/nginxd" + +service_config_dir="/etc/systemd/system" +service_name="jh_taosd" +tarbitrator_service_name="tarbitratord" +nginx_service_name="nginxd" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_process() { + pid=$(ps -ef | grep "jh_taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/jh_taos || : + ${csudo} rm -f ${bin_link_dir}/jh_taosd || : + ${csudo} rm -f ${bin_link_dir}/jhdemo || : + ${csudo} rm -f ${bin_link_dir}/jh_taosdump || : + ${csudo} rm -f ${bin_link_dir}/rmjh || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +function clean_service_on_systemd() { + service_config="${service_config_dir}/${service_name}.service" + if systemctl is-active --quiet ${service_name}; then + echo "jh_iot's jh_taosd is running, stopping it..." + ${csudo} systemctl stop ${service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${service_config} + + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "jh_iot's tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/${nginx_service_name}.service" + if [ -d ${bin_dir}/web ]; then + if systemctl is-active --quiet ${nginx_service_name}; then + echo "Nginx for jh_iot is running, stopping it..." + ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${nginx_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${nginx_service_config} + fi + fi +} + +function clean_service_on_sysvinit() { + if pidof jh_taosd &> /dev/null; then + echo "jh_iot's jh_taosd is running, stopping it..." + ${csudo} service jh_taosd stop || : + fi + + if pidof tarbitrator &> /dev/null; then + echo "jh_iot's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/jh_taosd ]; then + ${csudo} chkconfig --del jh_taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/jh_taosd ]; then + ${csudo} insserv -r jh_taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/jh_taosd ]; then + ${csudo} update-rc.d -f jh_taosd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/jh_taosd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + kill_process + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config +# Remove data link directory +${csudo} rm -rf ${data_link_dir} || : + +${csudo} rm -rf ${install_main_dir} +${csudo} rm -rf ${install_nginxd_dir} +if [[ -e /etc/os-release ]]; then + osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +else + osinfo="" +fi + +echo -e "${GREEN}jh_iot is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_kh.sh b/packaging/tools/remove_kh.sh new file mode 100755 index 0000000000000000000000000000000000000000..0055e043e74a3fa287cebbd325f83c7f8f98ca8a --- /dev/null +++ b/packaging/tools/remove_kh.sh @@ -0,0 +1,209 @@ +#!/bin/bash +# +# Script to stop the service and uninstall kinghistorian, but retain the config, data and log files. + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/kinghistorian" +data_link_dir="/usr/local/kinghistorian/data" +log_link_dir="/usr/local/kinghistorian/log" +cfg_link_dir="/usr/local/kinghistorian/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" +install_nginxd_dir="/usr/local/nginxd" + +service_config_dir="/etc/systemd/system" +service_name="khserver" +tarbitrator_service_name="tarbitratord" +nginx_service_name="nginxd" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_process() { + pid=$(ps -ef | grep "khserver" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/khclient || : + ${csudo} rm -f ${bin_link_dir}/khserver || : + ${csudo} rm -f ${bin_link_dir}/khdemo || : + ${csudo} rm -f ${bin_link_dir}/khdump || : + ${csudo} rm -f ${bin_link_dir}/rmkh || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +function clean_service_on_systemd() { + service_config="${service_config_dir}/${service_name}.service" + if systemctl is-active --quiet ${service_name}; then + echo "KingHistorian's khserver is running, stopping it..." + ${csudo} systemctl stop ${service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${service_config} + + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "KingHistorian's tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/${nginx_service_name}.service" + if [ -d ${bin_dir}/web ]; then + if systemctl is-active --quiet ${nginx_service_name}; then + echo "Nginx for KingHistorian is running, stopping it..." + ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${nginx_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${nginx_service_config} + fi + fi +} + +function clean_service_on_sysvinit() { + if pidof khserver &> /dev/null; then + echo "KingHistorian's khserver is running, stopping it..." + ${csudo} service khserver stop || : + fi + + if pidof tarbitrator &> /dev/null; then + echo "KingHistorian's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/khserver ]; then + ${csudo} chkconfig --del khserver || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/khserver ]; then + ${csudo} insserv -r khserver || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/khserver ]; then + ${csudo} update-rc.d -f khserver remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/khserver || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + kill_process + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config +# Remove data link directory +${csudo} rm -rf ${data_link_dir} || : + +${csudo} rm -rf ${install_main_dir} +${csudo} rm -rf ${install_nginxd_dir} +if [[ -e /etc/os-release ]]; then + osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +else + osinfo="" +fi + +echo -e "${GREEN}KingHistorian is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_power.sh b/packaging/tools/remove_power.sh index 816869cf444d8001e0c0aae30840d2c40a9e6af4..55abed87da7f9c7ba9375e002aa0a2ffd7fad1ed 100755 --- a/packaging/tools/remove_power.sh +++ b/packaging/tools/remove_power.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to stop the service and uninstall TDengine, but retain the config, data and log files. +# Script to stop the service and uninstall PowerDB, but retain the config, data and log files. set -e #set -x @@ -112,7 +112,7 @@ function clean_service_on_systemd() { tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" if systemctl is-active --quiet ${tarbitrator_service_name}; then - echo "TDengine tarbitrator is running, stopping it..." + echo "PowerDB tarbitrator is running, stopping it..." ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null @@ -122,7 +122,7 @@ function clean_service_on_systemd() { nginx_service_config="${service_config_dir}/${nginx_service_name}.service" if [ -d ${bin_dir}/web ]; then if systemctl is-active --quiet ${nginx_service_name}; then - echo "Nginx for TDengine is running, stopping it..." + echo "Nginx for PowerDB is running, stopping it..." ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable ${nginx_service_name} &> /dev/null || echo &> /dev/null @@ -133,9 +133,6 @@ function clean_service_on_systemd() { } function clean_service_on_sysvinit() { - #restart_config_str="power:2345:respawn:${service_config_dir}/powerd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof powerd &> /dev/null; then echo "PowerDB powerd is running, stopping it..." ${csudo} service powerd stop || : @@ -183,7 +180,6 @@ function clean_service() { elif ((${service_mod}==1)); then clean_service_on_sysvinit else - # must manual stop taosd kill_powerd kill_tarbitrator fi diff --git a/packaging/tools/remove_pro.sh b/packaging/tools/remove_pro.sh index f6dad22bc21b02a9d717d530c50bc19c5a718478..5906d60197a14a6cbb9862a9cddb278faafa1d7a 100755 --- a/packaging/tools/remove_pro.sh +++ b/packaging/tools/remove_pro.sh @@ -177,7 +177,6 @@ function clean_service() { elif ((${service_mod}==1)); then clean_service_on_sysvinit else - # must manual stop taosd kill_prodbs kill_tarbitrator fi diff --git a/packaging/tools/remove_tq.sh b/packaging/tools/remove_tq.sh index 211eed4dff09ab5da00d5c475cd93148b5ce1b24..14e7dd024d8b7b6c6b39567effb5ad6e8a20f8d5 100755 --- a/packaging/tools/remove_tq.sh +++ b/packaging/tools/remove_tq.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to stop the service and uninstall TDengine, but retain the config, data and log files. +# Script to stop the service and uninstall TQ, but retain the config, data and log files. set -e #set -x @@ -112,7 +112,7 @@ function clean_service_on_systemd() { tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" if systemctl is-active --quiet ${tarbitrator_service_name}; then - echo "TDengine tarbitrator is running, stopping it..." + echo "TQ tarbitrator is running, stopping it..." ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null @@ -122,7 +122,7 @@ function clean_service_on_systemd() { nginx_service_config="${service_config_dir}/${nginx_service_name}.service" if [ -d ${bin_dir}/web ]; then if systemctl is-active --quiet ${nginx_service_name}; then - echo "Nginx for TDengine is running, stopping it..." + echo "Nginx for TQ is running, stopping it..." ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable ${nginx_service_name} &> /dev/null || echo &> /dev/null @@ -183,7 +183,6 @@ function clean_service() { elif ((${service_mod}==1)); then clean_service_on_sysvinit else - # must manual stop taosd kill_tqd kill_tarbitrator fi @@ -212,16 +211,5 @@ else osinfo="" fi -#if echo $osinfo | grep -qwi "ubuntu" ; then -## echo "this is ubuntu system" -# ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : -#elif echo $osinfo | grep -qwi "debian" ; then -## echo "this is debian system" -# ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : -#elif echo $osinfo | grep -qwi "centos" ; then -## echo "this is centos system" -# ${csudo} rpm -e --noscripts tdengine || : -#fi - echo -e "${GREEN}TQ is removed successfully!${NC}" echo diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 76978c94b2b344a47b60aa143cb37562e45e5ed9..4838f7ee904af1596fe7e1d89e45dc820863c30f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.2.1.0' +version: '2.2.2.4' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.2.1.0 + - usr/lib/libtaos.so.2.2.2.4 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0d06e5d39c0ed1916e0c2af7ccce5918e31ac42f..c8796479982b71aca6fd41c679c7da9e059727f5 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -32,7 +32,9 @@ IF (TD_LINUX) #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) - ADD_SUBDIRECTORY(tests) + IF (NOT TD_NINGSI_60) + ADD_SUBDIRECTORY(tests) + ENDIF () ELSEIF (TD_DARWIN) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) @@ -53,8 +55,9 @@ ELSEIF (TD_DARWIN) #SOVERSION dylib version #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) - - ADD_SUBDIRECTORY(tests) + IF (NOT TD_NINGSI_60) + ADD_SUBDIRECTORY(tests) + ENDIF () ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b250eed5a12e88b1c4829ceee52520764d4e96b6..38550e6f08b1ab7f947b8c2c8904d35170b5548b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -108,6 +108,7 @@ typedef struct SBlockKeyInfo { int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len); int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); +int32_t tscCreateDataBlockData(STableDataBlocks* dataBuf, size_t defaultSize, int32_t rowSize, int32_t startOffset); void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf); int tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index de439e64036999da0c9f6142ace7ebe1b847adbc..eb128237c5a9a17fbd027a7f602ad83bf256ec2c 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -115,8 +115,9 @@ typedef struct SParsedDataColInfo { int16_t numOfCols; int16_t numOfBound; uint16_t flen; // TODO: get from STSchema - uint16_t allNullLen; // TODO: get from STSchema + uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) uint16_t extendedVarLen; + uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) int32_t * boundedColumns; // bound column idx according to schema SBoundColumn * cols; SBoundIdxInfo *colIdxInfo; @@ -132,7 +133,7 @@ typedef struct { typedef struct { uint8_t memRowType; // default is 0, that is SDataRow uint8_t compareStat; // 0 no need, 1 need compare - TDRowTLenT kvRowInitLen; + int32_t rowSize; SMemRowInfo *rowInfo; } SMemRowBuilder; @@ -143,8 +144,7 @@ typedef enum { int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec); -int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, - int32_t allNullLen); +int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo); void destroyMemRowBuilder(SMemRowBuilder *pBuilder); /** @@ -529,16 +529,6 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen; } -static FORCE_INLINE void checkAndConvertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) { - if (isDataRow(row)) { - if (kvLen < (dataLen * KVRatioConvert)) { - memRowSetConvert(row); - } - } else if (kvLen > dataLen) { - memRowSetConvert(row); - } -} - static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) { memRowSetType(row, memRowType); if (isDataRowT(memRowType)) { @@ -638,8 +628,7 @@ static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str, - bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId, - int32_t *dataLen, int32_t *kvLen, uint8_t compareStat) { + bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId) { int64_t iv; int32_t ret; char * endptr = NULL; @@ -651,26 +640,22 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { // bool if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - tscAppendMemRowColValEx(row, &TRUE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &TRUE_VALUE, true, colId, pSchema->type, toffset); } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - tscAppendMemRowColValEx(row, &FALSE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, &FALSE_VALUE, true, colId, pSchema->type, toffset); } else { return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_INTEGER) { iv = strtoll(pToken->z, NULL, 10); - tscAppendMemRowColValEx(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, - dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset); } else if (pToken->type == TK_FLOAT) { double dv = strtod(pToken->z, NULL); - tscAppendMemRowColValEx(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, - dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset); } else { return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); } @@ -680,8 +665,7 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok case TSDB_DATA_TYPE_TINYINT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -691,15 +675,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } uint8_t tmpVal = (uint8_t)iv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_UTINYINT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -709,15 +692,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } uint8_t tmpVal = (uint8_t)iv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_SMALLINT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -727,15 +709,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } int16_t tmpVal = (int16_t)iv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_USMALLINT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -745,15 +726,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } uint16_t tmpVal = (uint16_t)iv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_INT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -763,15 +743,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } int32_t tmpVal = (int32_t)iv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_UINT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -781,15 +760,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } uint32_t tmpVal = (uint32_t)iv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_BIGINT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -798,14 +776,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); } - tscAppendMemRowColValEx(row, &iv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &iv, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_UBIGINT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -815,14 +792,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } uint64_t tmpVal = (uint64_t)iv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_FLOAT: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -835,14 +811,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok } float tmpVal = (float)dv; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_DOUBLE: if (isNullStr(pToken)) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -853,15 +828,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); } - tscAppendMemRowColValEx(row, &dv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &dv, true, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_BINARY: // binary data cannot be null-terminated char string, otherwise the last char of the string is lost if (pToken->type == TK_NULL) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { // too long values will return invalid sql, not be truncated automatically if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); @@ -869,14 +843,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); char *rowEnd = memRowEnd(row); STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n); - tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset); } break; case TSDB_DATA_TYPE_NCHAR: if (pToken->type == TK_NULL) { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; @@ -888,7 +861,7 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok return tscInvalidOperationMsg(msg, buf, pToken->z); } varDataSetLen(rowEnd, output); - tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset); } break; @@ -897,17 +870,16 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok if (primaryKey) { // When building SKVRow primaryKey, we should not skip even with NULL value. int64_t tmpVal = 0; - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } else { - tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); + tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); } } else { int64_t tmpVal; if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) { return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); } - tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); } break; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 82be24cff3aa983cf9d1235e07962f0cc04275d2..3e636cdc7bb14dd2c9e7a9a3f474c1105bdabff2 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -236,7 +236,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { return; } - if (pRes->qId == 0) { + if (pRes->qId == 0 && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { tscError("qhandle is invalid"); pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; tscAsyncResultOnError(pSql); diff --git a/src/client/src/tscGlobalmerge.c b/src/client/src/tscGlobalmerge.c index 130abdf4a73ade3951070a100555d15270c265f2..83822b6d3dd21cf63d9ad6ae0861ed1396b62e45 100644 --- a/src/client/src/tscGlobalmerge.c +++ b/src/client/src/tscGlobalmerge.c @@ -33,12 +33,12 @@ typedef struct SCompareParam { int32_t groupOrderType; } SCompareParam; -static bool needToMerge(SSDataBlock* pBlock, SArray* columnIndexList, int32_t index, char **buf) { +static bool needToMerge(SSDataBlock* pBlock, SArray* columnIndexList, int32_t tsc_index, char **buf) { int32_t ret = 0; size_t size = taosArrayGetSize(columnIndexList); if (size > 0) { - ret = compare_aRv(pBlock, columnIndexList, (int32_t) size, index, buf, TSDB_ORDER_ASC); + ret = compare_aRv(pBlock, columnIndexList, (int32_t) size, tsc_index, buf, TSDB_ORDER_ASC); } // if ret == 0, means the result belongs to the same group @@ -555,9 +555,9 @@ static void savePrevOrderColumns(char** prevRow, SArray* pColumnList, SSDataBloc int32_t size = (int32_t) taosArrayGetSize(pColumnList); for(int32_t i = 0; i < size; ++i) { - SColIndex* index = taosArrayGet(pColumnList, i); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, index->colIndex); - assert(index->colId == pColInfo->info.colId); + SColIndex* tsc_index = taosArrayGet(pColumnList, i); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, tsc_index->colIndex); + assert(tsc_index->colId == pColInfo->info.colId); memcpy(prevRow[i], pColInfo->pData + pColInfo->info.bytes * rowIndex, pColInfo->info.bytes); } @@ -967,7 +967,6 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { if (pOperator->pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_DESC) { SWAP(w->skey, w->ekey, TSKEY); - assert(w->skey <= w->ekey); } } } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 07db18b498873f4a023d8ea76aadd7e76a4cd8d2..30e2c8be93fb39d68518639ed484237c802959e3 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -152,7 +152,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, int32_t noteColLength) { int32_t rowLen = 0; - SColumnIndex index = {0}; + SColumnIndex tsc_index = {0}; pSql->cmd.numOfCols = numOfCols; @@ -163,7 +163,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, tstrncpy(f.name, "Field", sizeof(f.name)); SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &tsc_index, TSDB_DATA_TYPE_BINARY, (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, -1000, (TSDB_COL_NAME_LEN - 1), false); rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE); @@ -173,7 +173,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, tstrncpy(f.name, "Type", sizeof(f.name)); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE), + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &tsc_index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE), -1000, typeColLength, false); rowLen += typeColLength + VARSTR_HEADER_SIZE; @@ -183,7 +183,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, tstrncpy(f.name, "Length", sizeof(f.name)); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &tsc_index, TSDB_DATA_TYPE_INT, sizeof(int32_t), -1000, sizeof(int32_t), false); rowLen += sizeof(int32_t); @@ -193,7 +193,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, tstrncpy(f.name, "Note", sizeof(f.name)); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE), + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &tsc_index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE), -1000, noteColLength, false); rowLen += noteColLength + VARSTR_HEADER_SIZE; @@ -358,9 +358,13 @@ static int32_t tscGetTableTagValue(SCreateBuilder *builder, char *result) { int num_fields = taos_num_fields(pSql); TAOS_FIELD *fields = taos_fetch_fields(pSql); - char buf[TSDB_COL_NAME_LEN + 16]; for (int i = 0; i < num_fields; i++) { - memset(buf, 0, sizeof(buf)); + char *buf = calloc(1, lengths[i] + 1); + if (buf == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + memset(buf, 0, lengths[i] + 1); int32_t ret = tscGetNthFieldResult(row, fields, lengths, i, buf); if (i == 0) { @@ -373,10 +377,13 @@ static int32_t tscGetTableTagValue(SCreateBuilder *builder, char *result) { } else { snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s,", buf); } + + free(buf); + if (i == num_fields - 1) { sprintf(result + strlen(result) - 1, "%s", ")"); } - } + } if (0 == strlen(result)) { return TSDB_CODE_TSC_INVALID_TABLE_NAME; @@ -389,7 +396,7 @@ static int32_t tscGetTableTagValue(SCreateBuilder *builder, char *result) { static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const char *ddl) { int32_t rowLen = 0; int16_t ddlLen = (int16_t)strlen(ddl); - SColumnIndex index = {0}; + SColumnIndex tsc_index = {0}; pSql->cmd.numOfCols = 2; SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); @@ -407,7 +414,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const } SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false); + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &tsc_index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false); rowLen += f.bytes; @@ -420,7 +427,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const } pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &tsc_index, TSDB_DATA_TYPE_BINARY, (int16_t)(ddlLen + VARSTR_HEADER_SIZE), -1000, ddlLen, false); rowLen += ddlLen + VARSTR_HEADER_SIZE; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 0750abdf6ff991d8c19eb63b77c5ba9ff9ddfb03..49affc9040886efa95d044ada94233c7819ca841 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -41,9 +41,8 @@ enum { static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows); static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, char *str, char **end); -int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, - int32_t allNullLen) { - ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols)); +int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo) { + ASSERT(nRows >= 0 && pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols)); if (nRows > 0) { // already init(bind multiple rows by single column) if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) { @@ -51,48 +50,19 @@ int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3 } } - // default compareStat is ROW_COMPARE_NO_NEED - if (nBoundCols == 0) { // file input - pBuilder->memRowType = SMEM_ROW_DATA; - return TSDB_CODE_SUCCESS; + uint32_t dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + pColInfo->allNullLen; + uint32_t kvLen = TD_MEM_ROW_KV_HEAD_SIZE + pColInfo->numOfBound * sizeof(SColIdx) + pColInfo->boundNullLen; + if (isUtilizeKVRow(kvLen, dataLen)) { + pBuilder->memRowType = SMEM_ROW_KV; } else { - float boundRatio = ((float)nBoundCols / (float)nCols); - - if (boundRatio < KVRatioKV) { - pBuilder->memRowType = SMEM_ROW_KV; - return TSDB_CODE_SUCCESS; - } else if (boundRatio > KVRatioData) { - pBuilder->memRowType = SMEM_ROW_DATA; - return TSDB_CODE_SUCCESS; - } - pBuilder->compareStat = ROW_COMPARE_NEED; - - if (boundRatio < KVRatioPredict) { - pBuilder->memRowType = SMEM_ROW_KV; - } else { - pBuilder->memRowType = SMEM_ROW_DATA; - } - } - - pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx); - - if (nRows > 0) { - pBuilder->rowInfo = tcalloc(nRows, sizeof(SMemRowInfo)); - if (pBuilder->rowInfo == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int i = 0; i < nRows; ++i) { - (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen; - (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen; - } + pBuilder->memRowType = SMEM_ROW_DATA; } return TSDB_CODE_SUCCESS; } -int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { - int32_t index = 0; +int tsParseTime(SStrToken *pToken, int64_t *tsc_time, char **next, char *tsc_error, int16_t timePrec) { + int32_t tsc_index = 0; SStrToken sToken; int64_t interval; int64_t useconds = 0; @@ -106,8 +76,8 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 useconds = taosStr2int64(pToken->z); } else { // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm); - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(error, "invalid timestamp format", pToken->z); + if (taosParseTime(pToken->z, tsc_time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(tsc_error, "invalid timestamp format", pToken->z); } return TSDB_CODE_SUCCESS; @@ -117,7 +87,7 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 if (isspace(pToken->z[k])) continue; if (pToken->z[k] == ',') { *next = pTokenEnd; - *time = useconds; + *tsc_time = useconds; return 0; } @@ -125,21 +95,21 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 } /* - * time expression: + * tsc_time expression: * e.g., now+12a, now-5h */ SStrToken valueToken; - index = 0; - sToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; + tsc_index = 0; + sToken = tStrGetToken(pTokenEnd, &tsc_index, false); + pTokenEnd += tsc_index; if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) { - index = 0; - valueToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; + tsc_index = 0; + valueToken = tStrGetToken(pTokenEnd, &tsc_index, false); + pTokenEnd += tsc_index; if (valueToken.n < 2) { - return tscInvalidOperationMsg(error, "value expected in timestamp", sToken.z); + return tscInvalidOperationMsg(tsc_error, "value expected in timestamp", sToken.z); } char unit = 0; @@ -156,7 +126,7 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 *next = pTokenEnd; } - *time = useconds; + *tsc_time = useconds; return TSDB_CODE_SUCCESS; } @@ -448,7 +418,7 @@ int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, char *tmpTokenBuf, SInsertStatementParam *pInsertParam) { - int32_t index = 0; + int32_t tsc_index = 0; SStrToken sToken = {0}; char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header @@ -457,8 +427,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i STableMeta * pTableMeta = pDataBlocks->pTableMeta; SSchema * schema = tscGetTableSchema(pTableMeta); SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder; - int32_t dataLen = spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE; - int32_t kvLen = pBuilder->kvRowInitLen; bool isParseBindParam = false; initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); @@ -472,9 +440,9 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i SSchema *pSchema = &schema[colIndex]; // get colId here - index = 0; - sToken = tStrGetToken(*str, &index, true); - *str += index; + tsc_index = 0; + sToken = tStrGetToken(*str, &tsc_index, true); + *str += tsc_index; if (sToken.type == TK_QUESTION) { if (!isParseBindParam) { @@ -535,8 +503,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i int16_t colId = -1; tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); - int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, - colId, &dataLen, &kvLen, pBuilder->compareStat); + int32_t ret = + tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -551,13 +519,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i } if (!isParseBindParam) { - // 2. check and set convert flag - if (pBuilder->compareStat == ROW_COMPARE_NEED) { - checkAndConvertMemRow(row, dataLen, kvLen); - } - - // 3. set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) { + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && isDataRow(row)) { SDataRow dataRow = memRowDataBody(row); for (int32_t i = 0; i < spd->numOfCols; ++i) { if (spd->cols[i].valStat == VAL_STAT_NONE) { @@ -567,7 +530,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i } } - *len = getExtendedRowSize(pDataBlocks); + *len = pBuilder->rowSize; return TSDB_CODE_SUCCESS; } @@ -606,7 +569,7 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SInsertStatementParam *pInsertParam, int32_t* numOfRows, char *tmpTokenBuf) { - int32_t index = 0; + int32_t tsc_index = 0; int32_t code = 0; (*numOfRows) = 0; @@ -620,17 +583,17 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn int32_t extendedRowSize = getExtendedRowSize(pDataBlock); - if (TSDB_CODE_SUCCESS != - (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound, - pDataBlock->boundColumnInfo.allNullLen))) { + if (TSDB_CODE_SUCCESS != (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, &pDataBlock->boundColumnInfo))) { return code; } + pDataBlock->rowBuilder.rowSize = extendedRowSize; + while (1) { - index = 0; - sToken = tStrGetToken(*str, &index, false); + tsc_index = 0; + sToken = tStrGetToken(*str, &tsc_index, false); if (sToken.n == 0 || sToken.type != TK_LP) break; - *str += index; + *str += tsc_index; if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { int32_t tSize; code = tscAllocateMemIfNeed(pDataBlock, extendedRowSize, &tSize); @@ -651,13 +614,13 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn pDataBlock->size += len; - index = 0; - sToken = tStrGetToken(*str, &index, false); + tsc_index = 0; + sToken = tStrGetToken(*str, &tsc_index, false); if (sToken.n == 0 || sToken.type != TK_RP) { return tscSQLSyntaxErrMsg(pInsertParam->msg, ") expected", *str); } - *str += index; + *str += tsc_index; (*numOfRows)++; } @@ -703,6 +666,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32 pColInfo->boundedColumns[i] = i; } pColInfo->allNullLen += pColInfo->flen; + pColInfo->boundNullLen = pColInfo->allNullLen; // default set allNullLen pColInfo->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); } @@ -905,7 +869,7 @@ static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char } static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundColumn) { - int32_t index = 0; + int32_t tsc_index = 0; SStrToken sToken = {0}; SStrToken tableToken = {0}; int32_t code = TSDB_CODE_SUCCESS; @@ -920,14 +884,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC char *sql = *sqlstr; // get the token of specified table - index = 0; - tableToken = tStrGetToken(sql, &index, false); - sql += index; + tsc_index = 0; + tableToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; // skip possibly exists column list - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; int32_t numOfColList = 0; @@ -936,8 +900,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC *boundColumn = &sToken.z[0]; while (1) { - index = 0; - sToken = tStrGetToken(sql, &index, false); + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); if (sToken.type == TK_ILLEGAL) { return tscSQLSyntaxErrMsg(pCmd->payload, "unrecognized token", sToken.z); @@ -947,12 +911,12 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC break; } - sql += index; + sql += tsc_index; ++numOfColList; } - sToken = tStrGetToken(sql, &index, false); - sql += index; + sToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; } if (numOfColList == 0 && (*boundColumn) != NULL) { @@ -962,9 +926,9 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); if (sToken.type == TK_USING) { // create table if not exists according to the super table - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; //the source super table is moved to the secondary position of the pTableMetaInfo list if (pQueryInfo->numOfTables < 2) { @@ -995,8 +959,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC SParsedDataColInfo spd = {0}; tscSetBoundColumnInfo(&spd, pTagSchema, tscGetNumOfTags(pSTableMetaInfo->pTableMeta)); - index = 0; - sToken = tStrGetToken(sql, &index, false); + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); if (sToken.type != TK_TAGS && sToken.type != TK_LP) { tscDestroyBoundColumnInfo(&spd); return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword TAGS expected", sToken.z); @@ -1017,16 +981,16 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC sql = end; - index = 0; // keywords of "TAGS" - sToken = tStrGetToken(sql, &index, false); - sql += index; + tsc_index = 0; // keywords of "TAGS" + sToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; } else { - sql += index; + sql += tsc_index; } - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; if (sToken.type != TK_LP) { tscDestroyBoundColumnInfo(&spd); @@ -1042,9 +1006,9 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC for (int i = 0; i < spd.numOfBound; ++i) { SSchema* pSchema = &pTagSchema[spd.boundedColumns[i]]; - index = 0; - sToken = tStrGetToken(sql, &index, true); - sql += index; + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, true); + sql += tsc_index; if (TK_ILLEGAL == sToken.type) { tdDestroyKVRowBuilder(&kvRowBuilder); @@ -1096,9 +1060,9 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC free(row); pInsertParam->tagData.data = pTag; - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; if (sToken.n == 0 || sToken.type != TK_RP) { return tscSQLSyntaxErrMsg(pInsertParam->msg, ") expected", sToken.z); } @@ -1107,9 +1071,9 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC * insert into table_name using super_table(tag_name1, tag_name2) tags(tag_val1, tag_val2) * (normal_col1, normal_col2) values(normal_col1_val, normal_col2_val); * */ - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); + sql += tsc_index; int numOfColsAfterTags = 0; if (sToken.type == TK_LP) { if (*boundColumn != NULL) { @@ -1119,18 +1083,18 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } while (1) { - index = 0; - sToken = tStrGetToken(sql, &index, false); + tsc_index = 0; + sToken = tStrGetToken(sql, &tsc_index, false); if (sToken.type == TK_RP) { break; } - if (sToken.n == 0 || sToken.type == TK_SEMI || index == 0) { + if (sToken.n == 0 || sToken.type == TK_SEMI || tsc_index == 0) { return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected token", sql); } - sql += index; + sql += tsc_index; ++numOfColsAfterTags; } @@ -1138,7 +1102,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - sToken = tStrGetToken(sql, &index, false); + sToken = tStrGetToken(sql, &tsc_index, false); } sql = sToken.z; @@ -1206,6 +1170,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat int32_t nCols = pColInfo->numOfCols; pColInfo->numOfBound = 0; + pColInfo->boundNullLen = 0; memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColInfo->cols[i].valStat = VAL_STAT_NONE; @@ -1213,9 +1178,9 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat int32_t code = TSDB_CODE_SUCCESS; - int32_t index = 0; - SStrToken sToken = tStrGetToken(str, &index, false); - str += index; + int32_t tsc_index = 0; + SStrToken sToken = tStrGetToken(str, &tsc_index, false); + str += tsc_index; if (sToken.type != TK_LP) { code = tscSQLSyntaxErrMsg(pInsertParam->msg, "( is expected", sToken.z); @@ -1225,9 +1190,9 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat bool isOrdered = true; int32_t lastColIdx = -1; // last column found while (1) { - index = 0; - sToken = tStrGetToken(str, &index, false); - str += index; + tsc_index = 0; + sToken = tStrGetToken(str, &tsc_index, false); + str += tsc_index; if (TK_STRING == sToken.type) { tscDequoteAndTrimToken(&sToken); @@ -1255,6 +1220,17 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->cols[t].valStat = VAL_STAT_HAS; pColInfo->boundedColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; + switch (pSchema[t].type) { + case TSDB_DATA_TYPE_BINARY: + pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); + break; + case TSDB_DATA_TYPE_NCHAR: + pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + break; + default: + pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type]; + break; + } findColumnIndex = true; if (isOrdered && (lastColIdx > t)) { isOrdered = false; @@ -1278,6 +1254,17 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->cols[t].valStat = VAL_STAT_HAS; pColInfo->boundedColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; + switch (pSchema[t].type) { + case TSDB_DATA_TYPE_BINARY: + pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); + break; + case TSDB_DATA_TYPE_NCHAR: + pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + break; + default: + pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type]; + break; + } findColumnIndex = true; if (isOrdered && (lastColIdx > t)) { isOrdered = false; @@ -1378,8 +1365,8 @@ int tsParseInsertSql(SSqlObj *pSql) { tscDebug("0x%"PRIx64" create data block list hashList:%p", pSql->self, pInsertParam->pTableBlockHashList); while (1) { - int32_t index = 0; - SStrToken sToken = tStrGetToken(str, &index, false); + int32_t tsc_index = 0; + SStrToken sToken = tStrGetToken(str, &tsc_index, false); // no data in the sql string anymore. if (sToken.n == 0) { @@ -1437,9 +1424,9 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - index = 0; - sToken = tStrGetToken(str, &index, false); - str += index; + tsc_index = 0; + sToken = tStrGetToken(str, &tsc_index, false); + str += tsc_index; if (sToken.n == 0 || (sToken.type != TK_FILE && sToken.type != TK_VALUES)) { code = tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword VALUES or FILE required", sToken.z); @@ -1452,13 +1439,13 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - index = 0; - sToken = tStrGetToken(str, &index, false); + tsc_index = 0; + sToken = tStrGetToken(str, &tsc_index, false); if (sToken.type != TK_STRING && sToken.type != TK_ID) { code = tscSQLSyntaxErrMsg(pInsertParam->msg, "file path is required following keyword FILE", sToken.z); goto _clean; } - str += index; + str += tsc_index; if (sToken.n == 0) { code = tscSQLSyntaxErrMsg(pInsertParam->msg, "file path is required following keyword FILE", sToken.z); goto _clean; @@ -1553,10 +1540,10 @@ int tsInsertInitialCheck(SSqlObj *pSql) { return TSDB_CODE_TSC_NO_WRITE_AUTH; } - int32_t index = 0; + int32_t tsc_index = 0; SSqlCmd *pCmd = &pSql->cmd; - SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false); + SStrToken sToken = tStrGetToken(pSql->sqlstr, &tsc_index, false); assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); pCmd->count = 0; @@ -1566,7 +1553,7 @@ int tsInsertInitialCheck(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT); - sToken = tStrGetToken(pSql->sqlstr, &index, false); + sToken = tStrGetToken(pSql->sqlstr, &tsc_index, false); if (sToken.type != TK_INTO) { return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword INTO is expected", sToken.z); } @@ -1722,13 +1709,18 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow goto _error; } - tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(pTableDataBlock), &maxRows); + int32_t extendedRowSize = getExtendedRowSize(pTableDataBlock); + tscAllocateMemIfNeed(pTableDataBlock, extendedRowSize, &maxRows); tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); if (tokenBuf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } + // insert from .csv means full and ordered columns, thus use SDataRow all the time + ASSERT(SMEM_ROW_DATA == pTableDataBlock->rowBuilder.memRowType); + pTableDataBlock->rowBuilder.rowSize = extendedRowSize; + while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 8f1f21842015cecfdd71486673cba2dcc1dd73a5..e2f05b3e23aa9540fee5dc46094bffa5a57900fb 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -970,25 +970,25 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam taosArraySetSize(tagBinds, numTags); int isNullColBind = TSDB_TRUE; for (int j = 0; j < numTags; ++j) { - TAOS_BIND* bind = taosArrayGet(tagBinds, j); - bind->is_null = &isNullColBind; + TAOS_BIND* tsc_bind = taosArrayGet(tagBinds, j); + tsc_bind->is_null = &isNullColBind; } for (int j = 0; j < numTags; ++j) { if (tagKVs[j] == NULL) continue; TAOS_SML_KV* kv = tagKVs[j]; - TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); - bind->buffer_type = kv->type; - bind->length = malloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; + TAOS_BIND* tsc_bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); + tsc_bind->buffer_type = kv->type; + tsc_bind->length = malloc(sizeof(uintptr_t*)); + *tsc_bind->length = kv->length; + tsc_bind->buffer = kv->value; + tsc_bind->is_null = NULL; } int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { - TAOS_BIND* bind = taosArrayGet(tagBinds, i); - free(bind->length); + TAOS_BIND* tsc_bind = taosArrayGet(tagBinds, i); + free(tsc_bind->length); } taosArrayDestroy(tagBinds); return code; @@ -1014,17 +1014,17 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, } for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - bind->is_null = &isNullColBind; + TAOS_BIND* tsc_bind = colBinds + j; + tsc_bind->is_null = &isNullColBind; } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; - TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; - bind->buffer_type = kv->type; - bind->length = malloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; + TAOS_BIND* tsc_bind = colBinds + kv->fieldSchemaIdx; + tsc_bind->buffer_type = kv->type; + tsc_bind->length = malloc(sizeof(uintptr_t*)); + *tsc_bind->length = kv->length; + tsc_bind->buffer = kv->value; + tsc_bind->is_null = NULL; } taosArrayPush(rowsBind, &colBinds); } @@ -1037,8 +1037,8 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, for (int i = 0; i < rows; ++i) { TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - free(bind->length); + TAOS_BIND* tsc_bind = colBinds + j; + free(tsc_bind->length); } free(colBinds); } @@ -1794,14 +1794,14 @@ static int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, return TSDB_CODE_SUCCESS; } -static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) { +static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **tsc_index, SSmlLinesInfo* info) { const char *start, *cur; int32_t ret = TSDB_CODE_SUCCESS; int len = 0; char key[] = "_ts"; char *value = NULL; - start = cur = *index; + start = cur = *tsc_index; *pTS = calloc(1, sizeof(TAOS_SML_KV)); while(*cur != '\0') { @@ -1851,8 +1851,8 @@ static bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { return false; } -static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; +static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **tsc_index, SHashObj *pHash, SSmlLinesInfo* info) { + const char *cur = *tsc_index; char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write uint16_t len = 0; @@ -1887,17 +1887,17 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash pKV->key = calloc(len + 1, 1); memcpy(pKV->key, key, len + 1); //tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); - *index = cur + 1; + *tsc_index = cur + 1; return TSDB_CODE_SUCCESS; } -static bool parseSmlValue(TAOS_SML_KV *pKV, const char **index, +static bool parseSmlValue(TAOS_SML_KV *pKV, const char **tsc_index, bool *is_last_kv, SSmlLinesInfo* info) { const char *start, *cur; char *value = NULL; uint16_t len = 0; - start = cur = *index; + start = cur = *tsc_index; while (1) { // unescaped ',' or ' ' or '\0' identifies a value @@ -1928,13 +1928,13 @@ static bool parseSmlValue(TAOS_SML_KV *pKV, const char **index, } free(value); - *index = (*cur == '\0') ? cur : cur + 1; + *tsc_index = (*cur == '\0') ? cur : cur + 1; return TSDB_CODE_SUCCESS; } -static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, +static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **tsc_index, uint8_t *has_tags, SSmlLinesInfo* info) { - const char *cur = *index; + const char *cur = *tsc_index; uint16_t len = 0; pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write @@ -1973,7 +1973,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index len++; } pSml->stableName[len] = '\0'; - *index = cur + 1; + *tsc_index = cur + 1; tscDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); return TSDB_CODE_SUCCESS; @@ -1992,10 +1992,10 @@ static int32_t isValidChildTableName(const char *pTbName, int16_t len) { static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, - const char **index, bool isField, + const char **tsc_index, bool isField, TAOS_SML_DATA_POINT* smlData, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; + const char *cur = *tsc_index; int32_t ret = TSDB_CODE_SUCCESS; TAOS_SML_KV *pkv; bool is_last_kv = false; @@ -2077,7 +2077,7 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, error: return ret; done: - *index = cur; + *tsc_index = cur; return ret; } @@ -2097,13 +2097,13 @@ static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *t } int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { - const char* index = sql; + const char* tsc_index = sql; int32_t ret = TSDB_CODE_SUCCESS; uint8_t has_tags = 0; TAOS_SML_KV *timestamp = NULL; SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - ret = parseSmlMeasurement(smlData, &index, &has_tags, info); + ret = parseSmlMeasurement(smlData, &tsc_index, &has_tags, info); if (ret) { tscError("SML:0x%"PRIx64" Unable to parse measurement", info->id); taosHashCleanup(keyHashTable); @@ -2113,7 +2113,7 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf //Parse Tags if (has_tags) { - ret = parseSmlKvPairs(&smlData->tags, &smlData->tagNum, &index, false, smlData, keyHashTable, info); + ret = parseSmlKvPairs(&smlData->tags, &smlData->tagNum, &tsc_index, false, smlData, keyHashTable, info); if (ret) { tscError("SML:0x%"PRIx64" Unable to parse tag", info->id); taosHashCleanup(keyHashTable); @@ -2123,7 +2123,7 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf tscDebug("SML:0x%"PRIx64" Parse tags finished, num of tags:%d", info->id, smlData->tagNum); //Parse fields - ret = parseSmlKvPairs(&smlData->fields, &smlData->fieldNum, &index, true, smlData, keyHashTable, info); + ret = parseSmlKvPairs(&smlData->fields, &smlData->fieldNum, &tsc_index, true, smlData, keyHashTable, info); if (ret) { tscError("SML:0x%"PRIx64" Unable to parse field", info->id); taosHashCleanup(keyHashTable); @@ -2133,7 +2133,7 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf taosHashCleanup(keyHashTable); //Parse timestamp - ret = parseSmlTimeStamp(×tamp, &index, info); + ret = parseSmlTimeStamp(×tamp, &tsc_index, info); if (ret) { tscError("SML:0x%"PRIx64" Unable to parse timestamp", info->id); return ret; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 6fb5b3c8d1116674937bb5930b3082ced5cd4485..4bb61d2ed8fc7c3091d67b14fa10ffd881822009 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -48,12 +48,14 @@ typedef struct SMultiTbStmt { bool nameSet; bool tagSet; bool subSet; + bool tagColSet; uint64_t currentUid; char *sqlstr; uint32_t tbNum; SStrToken tbname; SStrToken stbname; SStrToken values; + SStrToken tagCols; SArray *tags; STableDataBlocks *lastBlock; SHashObj *pTableHash; @@ -117,11 +119,11 @@ static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_ return TSDB_CODE_SUCCESS; } -static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { +static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* tsc_bind) { SNormalStmt* normal = &stmt->normal; for (uint16_t i = 0; i < normal->numParams; ++i) { - TAOS_BIND* tb = bind + i; + TAOS_BIND* tb = tsc_bind + i; tVariant* var = normal->params + i; tVariantDestroy(var); @@ -379,8 +381,8 @@ int32_t fillTablesColumnsNull(SSqlObj* pSql) { //////////////////////////////////////////////////////////////////////////////// // functions for insertion statement preparation -static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { - if (bind->is_null != NULL && *(bind->is_null)) { +static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* tsc_bind, int32_t colNum) { + if (tsc_bind->is_null != NULL && *(tsc_bind->is_null)) { setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; } @@ -399,35 +401,35 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam } u; switch (param->type) { case TSDB_DATA_TYPE_BOOL: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: { - u.v1 = *(int8_t*)bind->buffer; + u.v1 = *(int8_t*)tsc_bind->buffer; if (u.v1==0 || u.v1==1) break; } break; case TSDB_DATA_TYPE_TINYINT: { - u.v1 = *(int8_t*)bind->buffer; + u.v1 = *(int8_t*)tsc_bind->buffer; if (u.v1==0 || u.v1==1) break; } break; case TSDB_DATA_TYPE_SMALLINT: { - u.v1 = (int8_t)*(int16_t*)bind->buffer; + u.v1 = (int8_t)*(int16_t*)tsc_bind->buffer; if (u.v1==0 || u.v1==1) break; } break; case TSDB_DATA_TYPE_INT: { - u.v1 = (int8_t)*(int32_t*)bind->buffer; + u.v1 = (int8_t)*(int32_t*)tsc_bind->buffer; if (u.v1==0 || u.v1==1) break; } break; case TSDB_DATA_TYPE_BIGINT: { - u.v1 = (int8_t)*(int64_t*)bind->buffer; + u.v1 = (int8_t)*(int64_t*)tsc_bind->buffer; if (u.v1==0 || u.v1==1) break; } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { // "0", "1" convertible - if (strncmp((const char*)bind->buffer, "0", *bind->length)==0) { + if (strncmp((const char*)tsc_bind->buffer, "0", *tsc_bind->length)==0) { u.v1 = 0; break; } - if (strncmp((const char*)bind->buffer, "1", *bind->length)==0) { + if (strncmp((const char*)tsc_bind->buffer, "1", *tsc_bind->length)==0) { u.v1 = 1; break; } @@ -444,27 +446,27 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } break; case TSDB_DATA_TYPE_TINYINT: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { - int8_t v = *(int8_t*)bind->buffer; + int8_t v = *(int8_t*)tsc_bind->buffer; u.v1 = v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; } break; case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = *(int16_t*)bind->buffer; + int16_t v = *(int16_t*)tsc_bind->buffer; u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; } case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)bind->buffer; + int32_t v = *(int32_t*)tsc_bind->buffer; u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; } case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; + int64_t v = *(int64_t*)tsc_bind->buffer; u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; @@ -473,8 +475,8 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam case TSDB_DATA_TYPE_NCHAR: { int64_t v; int n, r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r == 1 && n == strlen((const char*)bind->buffer)) { + r = sscanf((const char*)tsc_bind->buffer, "%" PRId64 "%n", &v, &n); + if (r == 1 && n == strlen((const char*)tsc_bind->buffer)) { u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; } @@ -491,21 +493,21 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } case TSDB_DATA_TYPE_SMALLINT: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: { - int v = *(int16_t*)bind->buffer; + int v = *(int16_t*)tsc_bind->buffer; u.v2 = (int16_t)v; } break; case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)bind->buffer; + int32_t v = *(int32_t*)tsc_bind->buffer; u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; } case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; + int64_t v = *(int64_t*)tsc_bind->buffer; u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; @@ -514,8 +516,8 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam case TSDB_DATA_TYPE_NCHAR: { int64_t v; int n, r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r == 1 && n == strlen((const char*)bind->buffer)) { + r = sscanf((const char*)tsc_bind->buffer, "%" PRId64 "%n", &v, &n); + if (r == 1 && n == strlen((const char*)tsc_bind->buffer)) { u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; } @@ -532,15 +534,15 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } case TSDB_DATA_TYPE_INT: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: { - u.v4 = *(int32_t*)bind->buffer; + u.v4 = *(int32_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; + int64_t v = *(int64_t*)tsc_bind->buffer; u.v4 = (int32_t)v; if (v >= INT_MIN && v <= INT_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; @@ -549,8 +551,8 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam case TSDB_DATA_TYPE_NCHAR: { int64_t v; int n,r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + r = sscanf((const char*)tsc_bind->buffer, "%" PRId64 "%n", &v, &n); + if (r==1 && n==strlen((const char*)tsc_bind->buffer)) { u.v4 = (int32_t)v; if (v >= INT_MIN && v <= INT_MAX) break; } @@ -567,35 +569,35 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } break; case TSDB_DATA_TYPE_FLOAT: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { - u.f4 = *(int8_t*)bind->buffer; + u.f4 = *(int8_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_SMALLINT: { - u.f4 = *(int16_t*)bind->buffer; + u.f4 = *(int16_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_INT: { - u.f4 = (float)*(int32_t*)bind->buffer; + u.f4 = (float)*(int32_t*)tsc_bind->buffer; // shall we check equality? } break; case TSDB_DATA_TYPE_BIGINT: { - u.f4 = (float)*(int64_t*)bind->buffer; + u.f4 = (float)*(int64_t*)tsc_bind->buffer; // shall we check equality? } break; case TSDB_DATA_TYPE_FLOAT: { - u.f4 = *(float*)bind->buffer; + u.f4 = *(float*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_DOUBLE: { - u.f4 = *(float*)bind->buffer; + u.f4 = *(float*)tsc_bind->buffer; // shall we check equality? } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { float v; int n,r; - r = sscanf((const char*)bind->buffer, "%f%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + r = sscanf((const char*)tsc_bind->buffer, "%f%n", &v, &n); + if (r==1 && n==strlen((const char*)tsc_bind->buffer)) { u.f4 = v; break; } @@ -610,26 +612,26 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } break; case TSDB_DATA_TYPE_BIGINT: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { - u.v8 = *(int8_t*)bind->buffer; + u.v8 = *(int8_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_SMALLINT: { - u.v8 = *(int16_t*)bind->buffer; + u.v8 = *(int16_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_INT: { - u.v8 = *(int32_t*)bind->buffer; + u.v8 = *(int32_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_BIGINT: { - u.v8 = *(int64_t*)bind->buffer; + u.v8 = *(int64_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { int64_t v; int n,r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + r = sscanf((const char*)tsc_bind->buffer, "%" PRId64 "%n", &v, &n); + if (r==1 && n==strlen((const char*)tsc_bind->buffer)) { u.v8 = v; break; } @@ -646,32 +648,32 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } case TSDB_DATA_TYPE_DOUBLE: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { - u.f8 = *(int8_t*)bind->buffer; + u.f8 = *(int8_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_SMALLINT: { - u.f8 = *(int16_t*)bind->buffer; + u.f8 = *(int16_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_INT: { - u.f8 = *(int32_t*)bind->buffer; + u.f8 = *(int32_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_BIGINT: { - u.f8 = (double)*(int64_t*)bind->buffer; + u.f8 = (double)*(int64_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_FLOAT: { - u.f8 = *(float*)bind->buffer; + u.f8 = *(float*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_DOUBLE: { - u.f8 = *(double*)bind->buffer; + u.f8 = *(double*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { double v; int n,r; - r = sscanf((const char*)bind->buffer, "%lf%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + r = sscanf((const char*)tsc_bind->buffer, "%lf%n", &v, &n); + if (r==1 && n==strlen((const char*)tsc_bind->buffer)) { u.f8 = v; break; } @@ -686,15 +688,15 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } case TSDB_DATA_TYPE_TIMESTAMP: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_TIMESTAMP: { - u.v8 = *(int64_t*)bind->buffer; + u.v8 = *(int64_t*)tsc_bind->buffer; } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { // is this the correct way to call taosParseTime? - int32_t len = (int32_t)*bind->length; - if (taosParseTime(bind->buffer, &u.v8, len, 3, tsDaylight) == TSDB_CODE_SUCCESS) { + int32_t len = (int32_t)*tsc_bind->length; + if (taosParseTime(tsc_bind->buffer, &u.v8, len, 3, tsDaylight) == TSDB_CODE_SUCCESS) { break; } return TSDB_CODE_TSC_INVALID_VALUE; @@ -714,13 +716,13 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam return TSDB_CODE_SUCCESS; } case TSDB_DATA_TYPE_BINARY: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_BINARY: { - if ((*bind->length) > (uintptr_t)param->bytes) { + if ((*tsc_bind->length) > (uintptr_t)param->bytes) { return TSDB_CODE_TSC_INVALID_VALUE; } - short size = (short)*bind->length; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); + short size = (short)*tsc_bind->length; + STR_WITH_SIZE_TO_VARSTR(data + param->offset, tsc_bind->buffer, size); return TSDB_CODE_SUCCESS; } case TSDB_DATA_TYPE_BOOL: @@ -738,10 +740,10 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam } } case TSDB_DATA_TYPE_NCHAR: { - switch (bind->buffer_type) { + switch (tsc_bind->buffer_type) { case TSDB_DATA_TYPE_NCHAR: { int32_t output = 0; - if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(tsc_bind->buffer, *tsc_bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { return TSDB_CODE_TSC_INVALID_VALUE; } varDataSetLen(data + param->offset, output); @@ -768,7 +770,7 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam } #endif - if (bind->buffer_type != param->type) { + if (tsc_bind->buffer_type != param->type) { tscError("column type mismatch"); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -778,39 +780,39 @@ static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParam case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)(data + param->offset) = *(uint8_t *)bind->buffer; + *(uint8_t *)(data + param->offset) = *(uint8_t *)tsc_bind->buffer; break; case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)(data + param->offset) = *(uint16_t *)bind->buffer; + *(uint16_t *)(data + param->offset) = *(uint16_t *)tsc_bind->buffer; break; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_FLOAT: - *(uint32_t *)(data + param->offset) = *(uint32_t *)bind->buffer; + *(uint32_t *)(data + param->offset) = *(uint32_t *)tsc_bind->buffer; break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: - *(uint64_t *)(data + param->offset) = *(uint64_t *)bind->buffer; + *(uint64_t *)(data + param->offset) = *(uint64_t *)tsc_bind->buffer; break; case TSDB_DATA_TYPE_BINARY: - if ((*bind->length) > (uintptr_t)param->bytes) { + if ((*tsc_bind->length) > (uintptr_t)param->bytes) { tscError("column length is too big"); return TSDB_CODE_TSC_INVALID_VALUE; } - size = (short)*bind->length; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); + size = (short)*tsc_bind->length; + STR_WITH_SIZE_TO_VARSTR(data + param->offset, tsc_bind->buffer, size); return TSDB_CODE_SUCCESS; case TSDB_DATA_TYPE_NCHAR: { int32_t output = 0; - if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(tsc_bind->buffer, *tsc_bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { tscError("convert nchar failed"); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -885,27 +887,27 @@ static int32_t insertStmtGenBlock(STscStmt* pStmt, STableDataBlocks** pBlock, ST } -static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MULTI_BIND* bind, int32_t rowNum) { - if (bind->buffer_type != param->type || !isValidDataType(param->type)) { +static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MULTI_BIND* tsc_bind, int32_t rowNum) { + if (tsc_bind->buffer_type != param->type || !isValidDataType(param->type)) { tscError("column mismatch or invalid"); return TSDB_CODE_TSC_INVALID_VALUE; } - if (IS_VAR_DATA_TYPE(param->type) && bind->length == NULL) { + if (IS_VAR_DATA_TYPE(param->type) && tsc_bind->length == NULL) { tscError("BINARY/NCHAR no length"); return TSDB_CODE_TSC_INVALID_VALUE; } - for (int i = 0; i < bind->num; ++i) { + for (int i = 0; i < tsc_bind->num; ++i) { char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * (rowNum + i); - if (bind->is_null != NULL && bind->is_null[i]) { + if (tsc_bind->is_null != NULL && tsc_bind->is_null[i]) { setNull(data + param->offset, param->type, param->bytes); continue; } if (!IS_VAR_DATA_TYPE(param->type)) { - memcpy(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, tDataTypes[param->type].bytes); + memcpy(data + param->offset, (char *)tsc_bind->buffer + tsc_bind->buffer_length * i, tDataTypes[param->type].bytes); if (param->offset == 0) { if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { @@ -914,21 +916,21 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } } } else if (param->type == TSDB_DATA_TYPE_BINARY) { - if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("binary length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); + if (tsc_bind->length[i] > (uintptr_t)param->bytes) { + tscError("binary length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)tsc_bind->length[i]); return TSDB_CODE_TSC_INVALID_VALUE; } - int16_t bsize = (short)bind->length[i]; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, bsize); + int16_t bsize = (short)tsc_bind->length[i]; + STR_WITH_SIZE_TO_VARSTR(data + param->offset, (char *)tsc_bind->buffer + tsc_bind->buffer_length * i, bsize); } else if (param->type == TSDB_DATA_TYPE_NCHAR) { - if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("nchar string length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); + if (tsc_bind->length[i] > (uintptr_t)param->bytes) { + tscError("nchar string length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)tsc_bind->length[i]); return TSDB_CODE_TSC_INVALID_VALUE; } int32_t output = 0; - if (!taosMbsToUcs4((char *)bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - tscError("convert nchar string to UCS4_LE failed:%s", (char*)((char *)bind->buffer + bind->buffer_length * i)); + if (!taosMbsToUcs4((char *)tsc_bind->buffer + tsc_bind->buffer_length * i, tsc_bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { + tscError("convert nchar string to UCS4_LE failed:%s", (char*)((char *)tsc_bind->buffer + tsc_bind->buffer_length * i)); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -939,7 +941,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU return TSDB_CODE_SUCCESS; } -static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { +static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* tsc_bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; @@ -991,7 +993,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { SParamInfo* param = &pBlock->params[j]; - int code = doBindParam(pBlock, data, param, &bind[param->idx], 1); + int code = doBindParam(pBlock, data, param, &tsc_bind[param->idx], 1); if (code != TSDB_CODE_SUCCESS) { tscDebug("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); @@ -1002,10 +1004,10 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { } -static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int colIdx) { +static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* tsc_bind, int colIdx) { SSqlCmd* pCmd = &stmt->pSql->cmd; STscStmt* pStmt = (STscStmt*)stmt; - int rowNum = bind->num; + int rowNum = tsc_bind->num; STableDataBlocks* pBlock = NULL; @@ -1059,15 +1061,15 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c if (colIdx == -1) { for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { SParamInfo* param = &pBlock->params[j]; - if (bind[param->idx].num != rowNum) { - tscError("0x%"PRIx64" param %d: num[%d:%d] not match", pStmt->pSql->self, param->idx, rowNum, bind[param->idx].num); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind row num mismatch"); + if (tsc_bind[param->idx].num != rowNum) { + tscError("0x%"PRIx64" param %d: num[%d:%d] not match", pStmt->pSql->self, param->idx, rowNum, tsc_bind[param->idx].num); + return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "tsc_bind row num mismatch"); } - int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); + int code = doBindBatchParam(pBlock, param, &tsc_bind[param->idx], pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); + return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "tsc_bind column type mismatch or invalid"); } } @@ -1075,7 +1077,7 @@ static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int c } else { SParamInfo* param = &pBlock->params[colIdx]; - int code = doBindBatchParam(pBlock, param, bind, pCmd->batchSize); + int code = doBindBatchParam(pBlock, param, tsc_bind, pCmd->batchSize); if (code != TSDB_CODE_SUCCESS) { tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); @@ -1162,7 +1164,7 @@ static int insertStmtExecute(STscStmt* stmt) { SSqlCmd* pCmd = &stmt->pSql->cmd; if (pCmd->batchSize == 0) { tscError("no records bind"); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "no records bind"); + return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "no records tsc_bind"); } if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) == 0) { @@ -1246,6 +1248,12 @@ static void insertBatchClean(STscStmt* pStmt) { pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); pCmd->insertParam.numOfTables = 0; + STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); + while(p) { + tfree((*p)->pData); + p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); + } + taosHashClear(pCmd->insertParam.pTableBlockHashList); tscFreeSqlResult(pSql); tscFreeSubobj(pSql); @@ -1298,8 +1306,8 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { return ret; } - int32_t index = 0; - SStrToken sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); + int32_t tsc_index = 0; + SStrToken sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); if (sToken.n == 0) { tscError("table is is expected, sql:%s", pCmd->insertParam.sql); return tscSQLSyntaxErrMsg(pCmd->payload, "table name is expected", pCmd->insertParam.sql); @@ -1319,7 +1327,7 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { pStmt->mtb.tagSet = true; - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); if (sToken.n > 0 && (sToken.type == TK_VALUES || sToken.type == TK_LP)) { return TSDB_CODE_SUCCESS; } @@ -1329,20 +1337,51 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { return tscSQLSyntaxErrMsg(pCmd->payload, "keywords USING is expected", sToken.z ? sToken.z : pCmd->insertParam.sql); } - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); if (sToken.n <= 0 || ((sToken.type != TK_ID) && (sToken.type != TK_STRING))) { tscError("invalid token, sql:%s", pCmd->insertParam.sql); return tscSQLSyntaxErrMsg(pCmd->payload, "invalid token", sToken.z ? sToken.z : pCmd->insertParam.sql); } pStmt->mtb.stbname = sToken; - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || sToken.type != TK_TAGS) { - tscError("keyword TAGS expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z ? sToken.z : pCmd->insertParam.sql); + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); + if (sToken.n <= 0 || ((sToken.type != TK_TAGS) && (sToken.type != TK_LP))) { + tscError("invalid token, sql:%s", pCmd->insertParam.sql); + return tscSQLSyntaxErrMsg(pCmd->payload, "invalid token", sToken.z ? sToken.z : pCmd->insertParam.sql); + } + + // ... (tag_col_list) TAGS(tag_val_list) ... + int32_t tagColsCnt = 0; + if (sToken.type == TK_LP) { + pStmt->mtb.tagColSet = true; + pStmt->mtb.tagCols = sToken; + int32_t tagColsStart = tsc_index; + while (1) { + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); + if (sToken.type == TK_ILLEGAL) { + return tscSQLSyntaxErrMsg(pCmd->payload, "unrecognized token", sToken.z); + } + if (sToken.type == TK_ID) { + ++tagColsCnt; + } + if (sToken.type == TK_RP) { + break; + } + } + if (tagColsCnt == 0) { + tscError("tag column list expected, sql:%s", pCmd->insertParam.sql); + return tscSQLSyntaxErrMsg(pCmd->payload, "tag column list expected", pCmd->insertParam.sql); + } + pStmt->mtb.tagCols.n = tsc_index - tagColsStart + 1; + + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); + if (sToken.n <= 0 || sToken.type != TK_TAGS) { + tscError("keyword TAGS expected, sql:%s", pCmd->insertParam.sql); + return tscSQLSyntaxErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z ? sToken.z : pCmd->insertParam.sql); + } } - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); if (sToken.n <= 0 || sToken.type != TK_LP) { tscError("( expected, sql:%s", pCmd->insertParam.sql); return tscSQLSyntaxErrMsg(pCmd->payload, "( expected", sToken.z ? sToken.z : pCmd->insertParam.sql); @@ -1353,7 +1392,7 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { int32_t loopCont = 1; while (loopCont) { - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); if (sToken.n <= 0) { tscError("unexpected sql end, sql:%s", pCmd->insertParam.sql); return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected sql end", pCmd->insertParam.sql); @@ -1379,7 +1418,12 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { return tscSQLSyntaxErrMsg(pCmd->payload, "no tags", pCmd->insertParam.sql); } - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); + if (tagColsCnt > 0 && taosArrayGetSize(pStmt->mtb.tags) != tagColsCnt) { + tscError("not match tags, sql:%s", pCmd->insertParam.sql); + return tscSQLSyntaxErrMsg(pCmd->payload, "not match tags", pCmd->insertParam.sql); + } + + sToken = tStrGetToken(pCmd->insertParam.sql, &tsc_index, false); if (sToken.n <= 0 || (sToken.type != TK_VALUES && sToken.type != TK_LP)) { tscError("sql error, sql:%s", pCmd->insertParam.sql); return tscSQLSyntaxErrMsg(pCmd->payload, "sql error", sToken.z ? sToken.z : pCmd->insertParam.sql); @@ -1401,7 +1445,13 @@ int stmtGenInsertStatement(SSqlObj* pSql, STscStmt* pStmt, const char* name, TAO int32_t j = 0; while (1) { - len = (size_t)snprintf(str, size - 1, "insert into %s using %.*s tags(", name, pStmt->mtb.stbname.n, pStmt->mtb.stbname.z); + if (pStmt->mtb.tagColSet) { + len = (size_t)snprintf(str, size - 1, "insert into %s using %.*s %.*s tags(", + name, pStmt->mtb.stbname.n, pStmt->mtb.stbname.z, pStmt->mtb.tagCols.n, pStmt->mtb.tagCols.z); + } else { + len = (size_t)snprintf(str, size - 1, "insert into %s using %.*s tags(", name, pStmt->mtb.stbname.n, pStmt->mtb.stbname.z); + } + if (len >= (size -1)) { size *= 2; free(str); @@ -1477,6 +1527,41 @@ int stmtGenInsertStatement(SSqlObj* pSql, STscStmt* pStmt, const char* name, TAO return TSDB_CODE_SUCCESS; } +int32_t stmtValidateValuesFields(SSqlCmd *pCmd, char * sql) { + int32_t loopCont = 1, index0 = 0, values = 0; + SStrToken sToken; + + while (loopCont) { + sToken = tStrGetToken(sql, &index0, false); + if (sToken.n <= 0) { + return TSDB_CODE_SUCCESS; + } + + switch (sToken.type) { + case TK_RP: + if (values) { + return TSDB_CODE_SUCCESS; + } + break; + case TK_VALUES: + values = 1; + break; + case TK_QUESTION: + case TK_LP: + break; + default: + if (values) { + tscError("only ? allowed in values"); + return tscInvalidOperationMsg(pCmd->payload, "only ? allowed in values", NULL); + } + break; + } + } + + return TSDB_CODE_SUCCESS; +} + + //////////////////////////////////////////////////////////////////////////////// // interface functions @@ -1581,6 +1666,11 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { STMT_RET(ret); } + ret = stmtValidateValuesFields(&pSql->cmd, pSql->sqlstr); + if (ret != TSDB_CODE_SUCCESS) { + STMT_RET(ret); + } + if (pStmt->multiTbInsert) { STMT_RET(TSDB_CODE_SUCCESS); } @@ -1637,6 +1727,13 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags STMT_RET(TSDB_CODE_TSC_APP_ERROR); } + if ((*t1)->pData == NULL) { + code = tscCreateDataBlockData(*t1, TSDB_PAYLOAD_SIZE, (*t1)->pTableMeta->tableInfo.rowSize, sizeof(SSubmitBlk)); + if (code != TSDB_CODE_SUCCESS) { + STMT_RET(code); + } + } + SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; pCmd->batchSize = pBlk->numOfRows; if (pBlk->numOfRows == 0) { @@ -1695,7 +1792,6 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags STMT_RET(TSDB_CODE_SUCCESS); } - if (pStmt->mtb.tagSet) { pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); } else { @@ -1762,7 +1858,6 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags STMT_RET(code); } - int taos_stmt_set_sub_tbname(TAOS_STMT* stmt, const char* name) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHECK @@ -1770,8 +1865,6 @@ int taos_stmt_set_sub_tbname(TAOS_STMT* stmt, const char* name) { return taos_stmt_set_tbname_tags(stmt, name, NULL); } - - int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHECK @@ -1779,7 +1872,6 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { return taos_stmt_set_tbname_tags(stmt, name, NULL); } - int taos_stmt_close(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt == NULL || pStmt->taos == NULL) { @@ -1819,20 +1911,20 @@ int taos_stmt_close(TAOS_STMT* stmt) { STMT_RET(TSDB_CODE_SUCCESS); } -int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { +int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* tsc_bind) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHECK if (pStmt->isInsert) { if (pStmt->multiTbInsert) { if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); + tscError("0x%"PRIx64" tsc_bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "tsc_bind param status error")); } } else { if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); + tscError("0x%"PRIx64" tsc_bind param status error, last:%d", pStmt->pSql->self, pStmt->last); + STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "tsc_bind param status error")); } } @@ -1840,19 +1932,18 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { tscDebug("tableId:%" PRIu64 ", try to bind one row", pStmt->mtb.currentUid); - STMT_RET(insertStmtBindParam(pStmt, bind)); + STMT_RET(insertStmtBindParam(pStmt, tsc_bind)); } else { - STMT_RET(normalStmtBindParam(pStmt, bind)); + STMT_RET(normalStmtBindParam(pStmt, tsc_bind)); } } - -int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { +int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* tsc_bind) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHECK - if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX) { + if (tsc_bind == NULL || tsc_bind->num <= 0 || tsc_bind->num > INT16_MAX) { tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "invalid bind param")); } @@ -1876,14 +1967,14 @@ int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { pStmt->last = STMT_BIND; - STMT_RET(insertStmtBindParamBatch(pStmt, bind, -1)); + STMT_RET(insertStmtBindParamBatch(pStmt, tsc_bind, -1)); } -int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx) { +int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* tsc_bind, int colIdx) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHECK - if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX || colIdx < 0) { + if (tsc_bind == NULL || tsc_bind->num <= 0 || tsc_bind->num > INT16_MAX || colIdx < 0) { tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "invalid bind param")); } @@ -1907,11 +1998,9 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, in pStmt->last = STMT_BIND_COL; - STMT_RET(insertStmtBindParamBatch(pStmt, bind, colIdx)); + STMT_RET(insertStmtBindParamBatch(pStmt, tsc_bind, colIdx)); } - - int taos_stmt_add_batch(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHECK @@ -2052,7 +2141,6 @@ int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) { } } - char *taos_stmt_errstr(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; @@ -2063,8 +2151,6 @@ char *taos_stmt_errstr(TAOS_STMT *stmt) { return taos_errstr(pStmt->pSql); } - - const char *taos_data_type(int type) { switch (type) { case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL"; @@ -2081,4 +2167,3 @@ const char *taos_data_type(int type) { default: return "UNKNOWN"; } } - diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4cfec94f704d2821e9400861a181369743067185..08fa89fbc06adca64131cf41898a4027c7e8967a 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -48,7 +48,7 @@ #define COLUMN_INDEX_INITIAL_VAL (-2) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } -#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) +#define COLUMN_INDEX_VALIDE(tsc_index) (((tsc_index).tableIndex >= 0) && ((tsc_index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) #define TBNAME_LIST_SEP "," typedef struct SColumnList { // todo refactor @@ -99,7 +99,7 @@ static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery); -static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql); +static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql, bool joinQuery); static int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); static int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema); @@ -303,21 +303,21 @@ static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { } static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) { - int64_t time = 0; + int64_t tsc_time = 0; strdequote(pVar->pz); char* seg = strnchr(pVar->pz, '-', pVar->nLen, false); if (seg != NULL) { - if (taosParseTime(pVar->pz, &time, pVar->nLen, precision, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(pVar->pz, &tsc_time, pVar->nLen, precision, tsDaylight) != TSDB_CODE_SUCCESS) { return -1; } } else { - if (tVariantDump(pVar, (char*)&time, TSDB_DATA_TYPE_BIGINT, true)) { + if (tVariantDump(pVar, (char*)&tsc_time, TSDB_DATA_TYPE_BIGINT, true)) { return -1; } } tVariantDestroy(pVar); - tVariantCreateFromBinary(pVar, (char*)&time, 0, TSDB_DATA_TYPE_BIGINT); + tVariantCreateFromBinary(pVar, (char*)&tsc_time, 0, TSDB_DATA_TYPE_BIGINT); return 0; } static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tVariant* pVar) { @@ -358,19 +358,28 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { // validate the out put field type for "UNION ALL" subclause static int32_t normalizeVarDataTypeLength(SSqlCmd* pCmd) { const char* msg1 = "columns in select clause not identical"; + const char* msg2 = "too many select clause siblings, at most 100 allowed"; + int32_t siblings = 0; int32_t diffSize = 0; // if there is only one element, the limit of clause is the limit of global result. SQueryInfo* pQueryInfo1 = pCmd->pQueryInfo; SQueryInfo* pSibling = pQueryInfo1->sibling; + // pQueryInfo1 itself + ++siblings; + while(pSibling != NULL) { int32_t ret = tscFieldInfoCompare(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo, &diffSize); if (ret != 0) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } + if (++siblings > 100) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + pSibling = pSibling->sibling; } @@ -1038,8 +1047,8 @@ static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo, SSql SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); - SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL, getNewResColId(pCmd)); + SColumnIndex tsc_index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &tsc_index, &s, TSDB_COL_NORMAL, getNewResColId(pCmd)); return TSDB_CODE_SUCCESS; } @@ -1168,17 +1177,17 @@ static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(col, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(col, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else if (index.columnIndex >= numOfCols) { + } else if (tsc_index.columnIndex >= numOfCols) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } @@ -1187,15 +1196,15 @@ static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); } - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, tsc_index.columnIndex); if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE || pSchema->type == TSDB_DATA_TYPE_NCHAR || pSchema->type == TSDB_DATA_TYPE_BINARY) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; + tscColumnListInsert(pQueryInfo->colList, tsc_index.columnIndex, pTableMeta->id.uid, pSchema); + SColIndex colIndex = { .colIndex = tsc_index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; taosArrayPush(pGroupExpr->columnInfo, &colIndex); pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; pQueryInfo->stateWindow = true; @@ -1234,11 +1243,11 @@ int32_t validateSessionNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode * pS return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(col, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(col, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (tsc_index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1776,9 +1785,9 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 } // expr string is set as the parameter of function - SColumnIndex index = {.tableIndex = tableIndex}; + SColumnIndex tsc_index = {.tableIndex = tableIndex}; - SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), + SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &tsc_index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(pCmd), sizeof(double), false); char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->exprToken.z; @@ -1925,9 +1934,9 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); // add the timestamp column into the output columns - SColumnIndex index = {0}; // primary timestamp column info + SColumnIndex tsc_index = {0}; // primary timestamp column info int32_t numOfCols = (int32_t)tscNumOfExprs(pQueryInfo); - tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); + tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &tsc_index, pSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols); pSupInfo->visible = false; @@ -2029,7 +2038,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS const char* msg6 = "not support distinct mixed with join"; const char* msg7 = "not support distinct mixed with groupby"; const char* msg8 = "not support distinct in nest query"; - const char* msg9 = "not support group by in block func"; + const char* msg9 = "invalid alias name"; // too many result columns not support order by in query if (taosArrayGetSize(pSelNodeList) > TSDB_MAX_COLUMNS) { @@ -2053,6 +2062,9 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS hasDistinct = (pItem->distinct == true); distIdx = hasDistinct ? i : -1; } + if(pItem->aliasName != NULL && validateColumnName(pItem->aliasName) != TSDB_CODE_SUCCESS){ + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); + } int32_t type = pItem->pNode->type; if (type == SQL_NODE_SQLFUNCTION) { @@ -2065,10 +2077,6 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (pItem->pNode->functionId == TSDB_FUNC_BLKINFO && pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - SUdfInfo* pUdfInfo = NULL; if (pItem->pNode->functionId < 0) { pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); @@ -2167,16 +2175,16 @@ SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tab SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - SColumnIndex index = {.tableIndex = tableIndex,}; + SColumnIndex tsc_index = {.tableIndex = tableIndex,}; if (functionId == TSDB_FUNC_TAGPRJ) { - index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema); + tsc_index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta); + tscColumnListInsert(pTableMetaInfo->tagColList, tsc_index.columnIndex, pTableMeta->id.uid, pSchema); } else { - index.columnIndex = colIndex; + tsc_index.columnIndex = colIndex; } - return tscExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, 0, + return tscExprAppend(pQueryInfo, functionId, &tsc_index, pSchema->type, pSchema->bytes, colId, 0, (functionId == TSDB_FUNC_TAGPRJ)); } @@ -2245,51 +2253,51 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t if (tokenId == TK_ALL) { // project on all fields TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getTableIndexByName(&pItem->pNode->columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getTableIndexByName(&pItem->pNode->columnName, pQueryInfo, &tsc_index) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } // all meters columns are required - if (index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. + if (tsc_index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - index.tableIndex = i; - int32_t inc = doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); + tsc_index.tableIndex = i; + int32_t inc = doAddProjectionExprAndResultFields(pQueryInfo, &tsc_index, startPos, pCmd); startPos += inc; } } else { - doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); + doAddProjectionExprAndResultFields(pQueryInfo, &tsc_index, startPos, pCmd); } // add the primary timestamp column even though it is not required by user - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[index.tableIndex]->pTableMeta; + STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[tsc_index.tableIndex]->pTableMeta; if (pTableMeta->tableType != TSDB_TEMP_TABLE) { tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMeta->id.uid); } } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { // simple column projection query - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; // user-specified constant value as a new result column - index.columnIndex = (pQueryInfo->udColumnId--); - index.tableIndex = 0; + tsc_index.columnIndex = (pQueryInfo->udColumnId--); + tsc_index.tableIndex = 0; SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); - SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, + SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &tsc_index, &colSchema, TSDB_COL_UDC, getNewResColId(pCmd)); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->base.numOfParams = 2; tVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); } else if (tokenId == TK_ID) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pItem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&pItem->pNode->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (outerQuery) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); bool existed = false; @@ -2297,7 +2305,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t for (int32_t i = 0; i < numOfCols; ++i) { if (strncasecmp(pSchema[i].name, TSQL_TBNAME_L, tListLen(pSchema[i].name)) == 0) { existed = true; - index.columnIndex = i; + tsc_index.columnIndex = i; break; } } @@ -2306,12 +2314,12 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - SSchema colSchema = pSchema[index.columnIndex]; + SSchema colSchema = pSchema[tsc_index.columnIndex]; char name[TSDB_COL_NAME_LEN] = {0}; getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, + /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &tsc_index, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); } else { SSchema colSchema = *tGetTbnameColumnSchema(); @@ -2319,23 +2327,23 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, + /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &tsc_index, &colSchema, TSDB_COL_TAG, getNewResColId(pCmd)); } } else { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { + if (tsc_index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - addProjectQueryCol(pQueryInfo, startPos, &index, pItem, getNewResColId(pCmd)); + addProjectQueryCol(pQueryInfo, startPos, &tsc_index, pItem, getNewResColId(pCmd)); pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } // add the primary timestamp column even though it is not required by user - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); if (!UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); } @@ -2398,16 +2406,16 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT if (pItem->aliasName != NULL) { tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); } else if (multiCols) { - char uname[TSDB_COL_NAME_LEN] = {0}; + char tsc_index[TSDB_COL_NAME_LEN] = {0}; int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); - tstrncpy(uname, pToken->z, len); + tstrncpy(tsc_index, pToken->z, len); if (tsKeepOriginalColumnName) { // keep the original column name - tstrncpy(name, uname, TSDB_COL_NAME_LEN); + tstrncpy(name, tsc_index, TSDB_COL_NAME_LEN); } else { int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1; char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1] = {0}; - snprintf(tmp, size, "%s(%s)", aAggs[functionId].name, uname); + snprintf(tmp, size, "%s(%s)", aAggs[functionId].name, tsc_index); tstrncpy(name, tmp, TSDB_COL_NAME_LEN); } @@ -2474,7 +2482,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } SExprInfo* pExpr = NULL; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->Expr.paramList != NULL) { tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); @@ -2489,43 +2497,43 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // check if the table name is valid or not SStrToken tmpToken = pParamElem->pNode->columnName; - if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + if (getTableIndexByName(&tmpToken, pQueryInfo, &tsc_index) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tsc_index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); + pExpr = tscExprAppend(pQueryInfo, functionId, &tsc_index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); } else { // count the number of table created according to the super table - if (getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(pToken, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); // count tag is equalled to count(tbname) bool isTag = false; - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - index.columnIndex = TSDB_TBNAME_COLUMN_INDEX; + if (tsc_index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta) || tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + tsc_index.columnIndex = TSDB_TBNAME_COLUMN_INDEX; isTag = true; } int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, isTag); + pExpr = tscExprAppend(pQueryInfo, functionId, &tsc_index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, isTag); } } else { // count(*) is equalled to count(primary_timestamp_key) - index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tsc_index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); + pExpr = tscExprAppend(pQueryInfo, functionId, &tsc_index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - SColumnList list = createColumnList(1, index.tableIndex, index.columnIndex); + SColumnList list = createColumnList(1, tsc_index.tableIndex, tsc_index.columnIndex); if (finalResult) { int32_t numOfOutput = tscNumOfFields(pQueryInfo); insertResultField(pQueryInfo, numOfOutput, &list, sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, pExpr->base.aliasName, pExpr); @@ -2538,7 +2546,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } // the time stamp may be always needed - if (index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { + if (tsc_index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); } @@ -2571,21 +2579,21 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if ((getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if ((getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); STableComInfo info = tscGetTableInfo(pTableMetaInfo->pTableMeta); // functions can not be applied to tags - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta))) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || (tsc_index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta))) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // 2. check if sql function can be applied on this column data type - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, tsc_index.columnIndex); if (!IS_NUMERIC_TYPE(pSchema->type)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -2604,7 +2612,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // set the first column ts for diff query if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; + SColumnIndex indexTS = {.tableIndex = tsc_index.tableIndex, .columnIndex = 0}; SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false); tstrncpy(pExpr->base.aliasName, aAggs[TSDB_FUNC_TS_DUMMY].name, sizeof(pExpr->base.aliasName)); @@ -2613,7 +2621,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); } - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &tsc_index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); if (functionId == TSDB_FUNC_LEASTSQR) { // set the leastsquares parameters char val[8] = {0}; @@ -2665,7 +2673,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); } - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); + SColumnList ids = createColumnList(1, tsc_index.tableIndex, tsc_index.columnIndex); memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); @@ -2714,49 +2722,49 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; if (pParamElem->pNode->tokenId == TK_ALL) { // select table.* SStrToken tmpToken = pParamElem->pNode->columnName; - if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + if (getTableIndexByName(&tmpToken, pQueryInfo, &tsc_index) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); char name[TSDB_COL_NAME_LEN] = {0}; for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { - index.columnIndex = j; + tsc_index.columnIndex = j; SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); - if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &tsc_index, finalResult, pUdfInfo) != 0) { return TSDB_CODE_TSC_INVALID_OPERATION; } } } else { - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); // functions can not be applied to tags - if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { + if ((tsc_index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (tsc_index.columnIndex < 0)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } char name[TSDB_COL_NAME_LEN] = {0}; - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, tsc_index.columnIndex); bool multiColOutput = taosArrayGetSize(pItem->pNode->Expr.paramList) > 1; setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->columnName, multiColOutput); - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult, pUdfInfo) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &tsc_index, finalResult, pUdfInfo) != 0) { return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -2775,13 +2783,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { - SColumnIndex index = {.tableIndex = j, .columnIndex = i}; + SColumnIndex tsc_index = {.tableIndex = j, .columnIndex = i}; char name[TSDB_COL_NAME_LEN] = {0}; SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)}; setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); - if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[tsc_index.columnIndex], cvtFunc, name, colIndex, &tsc_index, finalResult, pUdfInfo) != 0) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -2809,20 +2817,20 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, tsc_index.columnIndex); // functions can not be applied to tags - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { + if (tsc_index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -2864,7 +2872,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); colIndex += 1; // the first column is ts - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), interResult, false); + pExpr = tscExprAppend(pQueryInfo, functionId, &tsc_index, resultType, resultSize, getNewResColId(pCmd), interResult, false); tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true); @@ -2876,19 +2884,19 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo REFACTOR // set the first column ts for top/bottom query - SColumnIndex index1 = {index.tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + SColumnIndex index1 = {tsc_index.tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), 0, false); tstrncpy(pExpr->base.aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->base.aliasName)); const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; - SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); + SColumnList ids = createColumnList(1, tsc_index.tableIndex, TS_COLUMN_INDEX); insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); + pExpr = tscExprAppend(pQueryInfo, functionId, &tsc_index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t)); } @@ -2896,7 +2904,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); // todo refactor: tscColumnListInsert part - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); + SColumnList ids = createColumnList(1, tsc_index.tableIndex, tsc_index.columnIndex); if (finalResult) { insertResultField(pQueryInfo, colIndex, &ids, resultSize, (int8_t)resultType, pExpr->base.aliasName, pExpr); @@ -2922,45 +2930,45 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tSqlExprItem* pParamItem = taosArrayGet(pItem->pNode->Expr.paramList, 0); tSqlExpr* pParam = pParamItem->pNode; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParam->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pParam->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); // functions can not be applied to normal columns int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (index.columnIndex < numOfCols && index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex < numOfCols && tsc_index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (index.columnIndex > 0) { - index.columnIndex -= numOfCols; + if (tsc_index.columnIndex > 0) { + tsc_index.columnIndex -= numOfCols; } // 2. valid the column type int16_t colType = 0; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { colType = TSDB_DATA_TYPE_BINARY; } else { - colType = pSchema[index.columnIndex].type; + colType = pSchema[tsc_index.columnIndex].type; } if (colType == TSDB_DATA_TYPE_BOOL) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, - &pSchema[index.columnIndex]); + tscColumnListInsert(pTableMetaInfo->tagColList, tsc_index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, + &pSchema[tsc_index.columnIndex]); SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); SSchema s = {0}; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { s = *tGetTbnameColumnSchema(); } else { - s = pTagSchema[index.columnIndex]; + s = pTagSchema[tsc_index.columnIndex]; } int16_t bytes = 0; @@ -2974,7 +2982,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col s.bytes = bytes; TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG, getNewResColId(pCmd)); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &tsc_index, &s, TSDB_COL_TAG, getNewResColId(pCmd)); return TSDB_CODE_SUCCESS; } @@ -2985,8 +2993,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SColumnIndex index = {.tableIndex = 0, .columnIndex = 0,}; - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + SColumnIndex tsc_index = {.tableIndex = 0, .columnIndex = 0,}; + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); int32_t inter = 0; int16_t resType = 0; @@ -2997,10 +3005,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY, .bytes = bytes}; SExprInfo* pExpr = - tscExprInsert(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, resType, bytes, getNewResColId(pCmd), bytes, 0); + tscExprInsert(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &tsc_index, resType, bytes, getNewResColId(pCmd), bytes, 0); tstrncpy(pExpr->base.aliasName, s.name, sizeof(pExpr->base.aliasName)); - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); + SColumnList ids = createColumnList(1, tsc_index.tableIndex, tsc_index.columnIndex); insertResultField(pQueryInfo, 0, &ids, bytes, s.type, s.name, pExpr); pExpr->base.numOfParams = 1; @@ -3025,19 +3033,19 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); // functions can not be applied to tags - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { + if (tsc_index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -3046,23 +3054,20 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col int16_t bytes = 0; getResultDataInfo(TSDB_DATA_TYPE_INT, 4, functionId, 0, &resType, &bytes, &inter, 0, false, pUdfInfo); - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resType, bytes, getNewResColId(pCmd), inter, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &tsc_index, resType, bytes, getNewResColId(pCmd), inter, false); memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token, sizeof(pExpr->base.aliasName) - 1); - SSchema s = {0}; - s.type = (uint8_t)resType; - s.bytes = bytes; - s.colId = pExpr->base.colInfo.colId; + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, tsc_index.columnIndex); uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); + SColumnList ids = createColumnList(1, tsc_index.tableIndex, tsc_index.columnIndex); if (finalResult) { insertResultField(pQueryInfo, colIndex, &ids, pUdfInfo->resBytes, pUdfInfo->resType, pExpr->base.aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, uid, &s); + tscColumnListInsert(pQueryInfo->colList, tsc_index.columnIndex, uid, pSchema); } } tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); @@ -3080,9 +3085,9 @@ static SColumnList createColumnList(int32_t num, int16_t tableIndex, int32_t col SColumnList columnList = {0}; columnList.num = num; - int32_t index = num - 1; - columnList.ids[index].tableIndex = tableIndex; - columnList.ids[index].columnIndex = columnIndex; + int32_t tsc_index = num - 1; + columnList.ids[tsc_index].tableIndex = tableIndex; + columnList.ids[tsc_index].columnIndex = columnIndex; return columnList; } @@ -3108,8 +3113,8 @@ static bool isTablenameToken(SStrToken* token) { return (tmpToken.n == strlen(TSQL_TBNAME_L) && strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0); } -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; +static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t tsc_index, SStrToken* pToken) { + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, tsc_index)->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); @@ -3134,15 +3139,17 @@ int32_t doGetColumnIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColum const char* msg0 = "ambiguous column name"; const char* msg1 = "invalid column name"; + if (pToken->n == 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + if (isTablenameToken(pToken)) { pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; } else if (strlen(DEFAULT_PRIMARY_TIMESTAMP_COL_NAME) == pToken->n && strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; // just make runtime happy, need fix java test case InsertSpecialCharacterJniTest - } else if (pToken->n == 0) { - pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; // just make runtime happy, need fix java test case InsertSpecialCharacterJniTest } else { - // not specify the table name, try to locate the table index by column name + // not specify the table name, try to locate the table tsc_index by column name if (pIndex->tableIndex == COLUMN_INDEX_INITIAL_VAL) { for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { int16_t colIndex = doGetColumnIndex(pQueryInfo, i, pToken); @@ -3156,7 +3163,7 @@ int32_t doGetColumnIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColum } } } - } else { // table index is valid, get the column index + } else { // table tsc_index is valid, get the column tsc_index int16_t colIndex = doGetColumnIndex(pQueryInfo, pIndex->tableIndex, pToken); if (colIndex != COLUMN_INDEX_INITIAL_VAL) { pIndex->columnIndex = colIndex; @@ -3520,6 +3527,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool int32_t scalarUdf = 0; int32_t prjNum = 0; int32_t aggNum = 0; + int32_t countTbname = 0; size_t numOfExpr = tscNumOfExprs(pQueryInfo); assert(numOfExpr > 0); @@ -3570,9 +3578,13 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { return false; } + + if (functionId == TSDB_FUNC_COUNT && (pExpr1->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX || TSDB_COL_IS_TAG(pExpr1->base.colInfo.flag))) { + ++countTbname; + } } - aggNum = (int32_t)size - prjNum - aggUdf - scalarUdf; + aggNum = (int32_t)size - prjNum - aggUdf - scalarUdf - countTbname; assert(aggNum >= 0); @@ -3584,6 +3596,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool return false; } + if (countTbname && (prjNum > 0 || aggNum > 0 || scalarUdf > 0 || aggUdf > 0)) { + return false; + } + return true; } @@ -3636,36 +3652,36 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&token, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&token, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - tableIndex = index.tableIndex; - } else if (tableIndex != index.tableIndex) { + tableIndex = tsc_index.tableIndex; + } else if (tableIndex != tsc_index.tableIndex) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pSchema = tGetTbnameColumnSchema(); } else { - pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); + pSchema = tscGetTableColumnSchema(pTableMeta, tsc_index.columnIndex); } int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols); + bool groupTag = (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || tsc_index.columnIndex >= numOfCols); if (groupTag) { if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - int32_t relIndex = index.columnIndex; - if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { + int32_t relIndex = tsc_index.columnIndex; + if (tsc_index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { relIndex -= numOfCols; } @@ -3673,17 +3689,17 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); taosArrayPush(pGroupExpr->columnInfo, &colIndex); - index.columnIndex = relIndex; - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema); + tsc_index.columnIndex = relIndex; + tscColumnListInsert(pTableMetaInfo->tagColList, tsc_index.columnIndex, pTableMeta->id.uid, pSchema); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); + tscColumnListInsert(pQueryInfo->colList, tsc_index.columnIndex, pTableMeta->id.uid, pSchema); - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; + SColIndex colIndex = { .colIndex = tsc_index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); taosArrayPush(pGroupExpr->columnInfo, &colIndex); @@ -4026,12 +4042,12 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSq return getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId); } else { // handle leaf node - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - return extractColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr); + return extractColumnFilterInfo(pCmd, pQueryInfo, &tsc_index, pExpr, relOptr); } } @@ -4055,17 +4071,17 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS return checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr->pRight); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); + SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, tsc_index.columnIndex); - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); + assert(tsc_index.tableIndex >= 0 && tsc_index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[tsc_index.tableIndex]; if (*leftNode == NULL) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4076,9 +4092,9 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + tsc_index.columnIndex = tsc_index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema1->colId, pTableMetaInfo->pTableMeta->id.uid) < 0) { - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema1); + tscColumnListInsert(pTableMetaInfo->tagColList, tsc_index.columnIndex, pTableMeta->id.uid, pTagSchema1); atomic_add_fetch_32(&pTableMetaInfo->joinTagNum, 1); if (pTableMetaInfo->joinTagNum > 1) { @@ -4087,19 +4103,19 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS } } - int16_t leftIdx = index.tableIndex; + int16_t leftIdx = tsc_index.tableIndex; - index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pRight->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + tsc_index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pExpr->pRight->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); + SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, tsc_index.columnIndex); - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); + assert(tsc_index.tableIndex >= 0 && tsc_index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[tsc_index.tableIndex]; if (*rightNode == NULL) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4109,10 +4125,10 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMeta); + tsc_index.columnIndex = tsc_index.columnIndex - tscGetNumOfColumns(pTableMeta); if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema2->colId, pTableMeta->id.uid) < 0) { - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema2); + tscColumnListInsert(pTableMetaInfo->tagColList, tsc_index.columnIndex, pTableMeta->id.uid, pTagSchema2); atomic_add_fetch_32(&pTableMetaInfo->joinTagNum, 1); if (pTableMetaInfo->joinTagNum > 1) { @@ -4121,7 +4137,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS } } - int16_t rightIdx = index.tableIndex; + int16_t rightIdx = tsc_index.tableIndex; if (pTagSchema1->type != pTagSchema2->type) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -4162,21 +4178,21 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQuer return TSDB_CODE_TSC_INVALID_OPERATION; } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pExpr->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } // if column is timestamp, bool, binary, nchar, not support arithmetic, so return invalid sql - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; - SSchema* pSchema = tscGetTableSchema(pTableMeta) + index.columnIndex; + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex)->pTableMeta; + SSchema* pSchema = tscGetTableSchema(pTableMeta) + tsc_index.columnIndex; if ((pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) || (pSchema->type == TSDB_DATA_TYPE_BOOL) || (pSchema->type == TSDB_DATA_TYPE_BINARY) || (pSchema->type == TSDB_DATA_TYPE_NCHAR)) { return TSDB_CODE_TSC_INVALID_OPERATION; } - pList->ids[pList->num++] = index; + pList->ids[pList->num++] = tsc_index; } else if ((pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) || pExpr->tokenId == TK_NULL) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -4195,7 +4211,11 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQuer // Append the sqlExpr into exprList of pQueryInfo structure sequentially pExpr->functionId = isValidFunction(pExpr->Expr.operand.z, pExpr->Expr.operand.n); if (pExpr->functionId < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; + SUdfInfo* pUdfInfo = NULL; + pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pExpr->Expr.operand.z, pExpr->Expr.operand.n); + if (pUdfInfo == NULL) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "invalid function name"); + } } if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false, NULL) != TSDB_CODE_SUCCESS) { @@ -4436,7 +4456,7 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) { } // check for like expression -static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { +static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t tsc_index, char* msgBuf) { const char* msg1 = "wildcard string should be less than %d characters"; const char* msg2 = "illegal column name"; @@ -4451,7 +4471,7 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t } SSchema* pSchema = tscGetTableSchema(pTableMeta); - if ((!isTablenameToken(&pLeft->columnName)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) { + if ((!isTablenameToken(&pLeft->columnName)) && !IS_VAR_DATA_TYPE(pSchema[tsc_index].type)) { return invalidOperationMsg(msgBuf, msg2); } } @@ -4460,7 +4480,7 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t } static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t parentOptr, int32_t* tbIdx) { + int32_t* type, int32_t parentOptr, int32_t* tbIdx, bool joinQuery) { const char* msg1 = "table query cannot use tags filter"; const char* msg2 = "illegal column name"; const char* msg3 = "only one query time range allowed"; @@ -4468,22 +4488,23 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql const char* msg5 = "not support ordinary column join"; const char* msg6 = "only one query condition on tbname allowed"; const char* msg7 = "only in/like allowed in filter table name"; + const char* msg8 = "illegal condition expression"; tSqlExpr* pLeft = (*pExpr)->pLeft; tSqlExpr* pRight = (*pExpr)->pRight; int32_t ret = TSDB_CODE_SUCCESS; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - *tbIdx = index.tableIndex; + *tbIdx = tsc_index.tableIndex; assert(tSqlExprIsParentOfLeaf(*pExpr)); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; // validate the null expression @@ -4493,14 +4514,14 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } // validate the like expression - code = validateLikeExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); + code = validateLikeExpr(*pExpr, pTableMeta, tsc_index.columnIndex, tscGetErrorMsgPayload(pCmd)); if (code != TSDB_CODE_SUCCESS) { return code; } - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range - if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, tsc_index.columnIndex); + if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && tsc_index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range + if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &tsc_index)) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -4509,8 +4530,8 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY); pCondExpr->tsJoin = true; - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + assert(tsc_index.tableIndex >= 0 && tsc_index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); + SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[tsc_index.tableIndex]; if (*leftNode == NULL) { *leftNode = calloc(1, sizeof(SJoinNode)); if (*leftNode == NULL) { @@ -4518,17 +4539,17 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } } - int16_t leftIdx = index.tableIndex; + int16_t leftIdx = tsc_index.tableIndex; - if (getColumnIndexByName(&pRight->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&pRight->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (index.tableIndex < 0 || index.tableIndex >= TSDB_MAX_JOIN_TABLE_NUM) { + if (tsc_index.tableIndex < 0 || tsc_index.tableIndex >= TSDB_MAX_JOIN_TABLE_NUM) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[tsc_index.tableIndex]; if (*rightNode == NULL) { *rightNode = calloc(1, sizeof(SJoinNode)); if (*rightNode == NULL) { @@ -4536,7 +4557,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } } - int16_t rightIdx = index.tableIndex; + int16_t rightIdx = tsc_index.tableIndex; if ((*leftNode)->tsJoin == NULL) { (*leftNode)->tsJoin = taosArrayInit(2, sizeof(int16_t)); @@ -4560,14 +4581,14 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql *pExpr = NULL; // remove this expression *type = TSQL_EXPR_TS; - } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + } else if (tsc_index.columnIndex >= tscGetNumOfColumns(pTableMeta) || tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { // query on tags, check for tag query condition if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } // in case of in operator, keep it in a seprate attribute - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } @@ -4579,7 +4600,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql if (pCondExpr->pTableCond == NULL) { pCondExpr->pTableCond = *pExpr; pCondExpr->relType = parentOptr; - pCondExpr->tableCondIndex = index.tableIndex; + pCondExpr->tableCondIndex = tsc_index.tableIndex; } else { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -4588,7 +4609,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql *pExpr = NULL; } else { if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query - if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { + if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &tsc_index)) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -4607,8 +4628,12 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } else { // query on other columns *type = TSQL_EXPR_COLUMN; - if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); + if (pRight->tokenId == TK_ID) { + if (joinQuery) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); // other column cannot be served as the join column + } else { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); + } } ret = setExprToCond(&pCondExpr->pColumnCond, *pExpr, NULL, parentOptr, pCmd->payload); @@ -4619,7 +4644,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t parentOptr, int32_t *tbIdx) { + int32_t* type, int32_t parentOptr, int32_t *tbIdx, bool joinQuery) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } @@ -4643,12 +4668,12 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr int32_t rightTbIdx = 0; if (!tSqlExprIsParentOfLeaf(*pExpr)) { - int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId, &leftTbIdx); + int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId, &leftTbIdx, joinQuery); if (ret != TSDB_CODE_SUCCESS) { return ret; } - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId, &rightTbIdx); + ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId, &rightTbIdx, joinQuery); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -4670,7 +4695,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr *type = rightType; *tbIdx = (leftTbIdx == rightTbIdx) ? leftTbIdx : -1; - + return TSDB_CODE_SUCCESS; } @@ -4684,19 +4709,19 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr return TSDB_CODE_TSC_INVALID_OPERATION; } - return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr, tbIdx); + return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr, tbIdx, joinQuery); } static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, tSqlExpr** pOut, int32_t tableIndex) { if (tSqlExprIsParentOfLeaf(*pExpr)) { tSqlExpr* pLeft = (*pExpr)->pLeft; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return; } - if (index.tableIndex != tableIndex) { + if (tsc_index.tableIndex != tableIndex) { return; } @@ -4861,12 +4886,12 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE return getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight); } else { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); tSqlExpr* pRight = pExpr->pRight; @@ -4944,27 +4969,27 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pLeft->ColName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pLeft->ColName, pQueryInfo, &tsc_index) != TSDB_CODE_SUCCESS) { tscError("%p: invalid column name (left)", pQueryInfo); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); + tsc_index.columnIndex = tsc_index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, &index, &pSchema[index.columnIndex]); + tscColumnListInsert(pTableMetaInfo->tagColList, &tsc_index, &pSchema[tsc_index.columnIndex]); - if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pRight->ColName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pRight->ColName, pQueryInfo, &tsc_index) != TSDB_CODE_SUCCESS) { tscError("%p: invalid column name (right)", pQueryInfo); } - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); + tsc_index.columnIndex = tsc_index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, &index, &pSchema[index.columnIndex]); + tscColumnListInsert(pTableMetaInfo->tagColList, &tsc_index, &pSchema[tsc_index.columnIndex]); } } */ @@ -5083,10 +5108,10 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE size_t num = taosArrayGetSize(colList); for(int32_t j = 0; j < num; ++j) { SColIndex* pIndex = taosArrayGet(colList, j); - SColumnIndex index = {.tableIndex = i, .columnIndex = pIndex->colIndex - numOfCols}; + SColumnIndex tsc_index = {.tableIndex = i, .columnIndex = pIndex->colIndex - numOfCols}; SSchema* s = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, + tscColumnListInsert(pTableMetaInfo->tagColList, tsc_index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, &s[pIndex->colIndex]); } @@ -5210,11 +5235,11 @@ int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { } -int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) { +int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql, bool joinQuery) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - + const char* msg1 = "invalid expression"; const char* msg2 = "invalid filter expression"; @@ -5230,7 +5255,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq int32_t type = 0; int32_t tbIdx = 0; - if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId, &tbIdx)) != TSDB_CODE_SUCCESS) { + if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId, &tbIdx, joinQuery)) != TSDB_CODE_SUCCESS) { goto PARSE_WHERE_EXIT; } @@ -5248,7 +5273,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq if ((ret = getTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { return ret; } - + // 3. get the tag query condition if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr, pExpr)) != TSDB_CODE_SUCCESS) { @@ -5562,19 +5587,25 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { } int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema) { - const char* msg0 = "only support order by primary timestamp"; - const char* msg1 = "invalid column name"; - const char* msg2 = "order by primary timestamp, first tag or groupby column in groupby clause allowed"; - const char* msg3 = "invalid column in order by clause, only primary timestamp or first tag in groupby clause allowed"; - const char* msg4 = "orderby column must projected in subquery"; + const char* msg0 = "only one column allowed in orderby"; + const char* msg1 = "invalid column name in orderby clause"; + const char* msg2 = "too many order by columns"; + const char* msg3 = "only primary timestamp/tbname/first tag in groupby clause allowed"; + const char* msg4 = "only tag in groupby clause allowed in order clause"; + const char* msg5 = "only primary timestamp/column in top/bottom function allowed as order column"; + const char* msg6 = "only primary timestamp allowed as the second order column"; + const char* msg7 = "only primary timestamp/column in groupby clause allowed as order column"; + const char* msg8 = "only column in groupby clause allowed as order column"; + const char* msg9 = "orderby column must projected in subquery"; + const char* msg10 = "not support distinct mixed with order by"; + const char* msg11 = "not support order with udf"; + const char* msg12 = "order by tags not supported with diff/derivative/csum/mavg"; setDefaultOrderInfo(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pQueryInfo->distinct || pSqlNode->pSortOrder == NULL) { + if (pSqlNode->pSortOrder == NULL) { return TSDB_CODE_SUCCESS; } - char* pMsgBuf = tscGetErrorMsgPayload(pCmd); SArray* pSortOrder = pSqlNode->pSortOrder; @@ -5582,32 +5613,54 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq * for table query, there is only one or none order option is allowed, which is the * ts or values(top/bottom) order is supported. * - * for super table query, the order option must be less than 3. + * for super table query, the order option must be less than 3 and the second must be ts. + * + * order by has 5 situations + * 1. from stable group by tag1 order by tag1 [ts] + * 2. from stable group by tbname order by tbname [ts] + * 3. from stable/table group by column1 order by column1 + * 4. from stable/table order by ts + * 5. select stable/table top(column2,1) ... order by column2 */ size_t size = taosArrayGetSize(pSortOrder); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { if (size > 1) { return invalidOperationMsg(pMsgBuf, msg0); } } else { if (size > 2) { - return invalidOperationMsg(pMsgBuf, msg3); + return invalidOperationMsg(pMsgBuf, msg2); } } + if (size > 0 && pQueryInfo->distinct) { + return invalidOperationMsg(pMsgBuf, msg10); + } // handle the first part of order by tVariant* pVar = taosArrayGet(pSortOrder, 0); - // e.g., order by 1 asc, return directly with out further check. - if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { - return TSDB_CODE_SUCCESS; + if (pVar->nType != TSDB_DATA_TYPE_BINARY){ + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex indexColumn = COLUMN_INDEX_INITIALIZER; + bool udf = false; + + if (pQueryInfo->pUdfInfo && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + int32_t usize = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + + for (int32_t i = 0; i < usize; ++i) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); + if (pUdfInfo->funcType == TSDB_UDF_TYPE_SCALAR) { + udf = true; + break; + } + } + } if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query - if (getColumnIndexByName(&columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&columnName, pQueryInfo, &indexColumn, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(pMsgBuf, msg1); } @@ -5615,43 +5668,49 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq bool orderByTS = false; bool orderByGroupbyCol = false; - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - + if (indexColumn.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // order by tag1 + int32_t relTagIndex = indexColumn.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + // it is a tag column if (pQueryInfo->groupbyExpr.columnInfo == NULL) { - return invalidOperationMsg(pMsgBuf, msg2); + return invalidOperationMsg(pMsgBuf, msg4); } SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); if (relTagIndex == pColIndex->colIndex) { orderByTags = true; } - } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - orderByTags = true; - } - - if (PRIMARYKEY_TIMESTAMP_COL_INDEX == index.columnIndex) { + } else if (indexColumn.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { // order by tbname + // it is a tag column + if (pQueryInfo->groupbyExpr.columnInfo == NULL) { + return invalidOperationMsg(pMsgBuf, msg4); + } + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + if (TSDB_TBNAME_COLUMN_INDEX == pColIndex->colIndex) { + orderByTags = true; + } + }else if (indexColumn.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // order by ts orderByTS = true; - } - - SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; - if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { - SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - if (PRIMARYKEY_TIMESTAMP_COL_INDEX != index.columnIndex && pColIndex->colIndex == index.columnIndex) { - orderByGroupbyCol = true; + }else{ // order by normal column + SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; + if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { + SColIndex* pColIndex = taosArrayGet(columnInfo, 0); + if (pColIndex->colIndex == indexColumn.columnIndex) { + orderByGroupbyCol = true; + } } } if (!(orderByTags || orderByTS || orderByGroupbyCol) && !isTopBottomQuery(pQueryInfo)) { return invalidOperationMsg(pMsgBuf, msg3); - } else { // order by top/bottom result value column is not supported in case of interval query. - assert(!(orderByTags && orderByTS && orderByGroupbyCol)); } size_t s = taosArrayGetSize(pSortOrder); if (s == 1) { if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + if (tscIsDiffDerivQuery(pQueryInfo)) { + return invalidOperationMsg(pMsgBuf, msg12); + } + pQueryInfo->groupbyExpr.orderIndex = indexColumn.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->groupbyExpr.orderType = p1->sortOrder; @@ -5659,32 +5718,41 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; + pQueryInfo->order.orderColId = pSchema[indexColumn.columnIndex].colId; + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } } else if (isTopBottomQuery(pQueryInfo)) { - int32_t topBotIndex = tscGetTopBotQueryExprIndex(pQueryInfo); - assert(topBotIndex >= 1); /* order of top/bottom query in interval is not valid */ - SExprInfo* pExpr = tscExprGet(pQueryInfo, topBotIndex-1); + + int32_t pos = tscGetTopBotQueryExprIndex(pQueryInfo); + assert(pos > 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); assert(pExpr->base.functionId == TSDB_FUNC_TS); - pExpr = tscExprGet(pQueryInfo, topBotIndex); - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg2); + pExpr = tscExprGet(pQueryInfo, pos); + + if (pExpr->base.colInfo.colIndex != indexColumn.columnIndex && indexColumn.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidOperationMsg(pMsgBuf, msg5); } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = p1->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; + pQueryInfo->order.orderColId = pSchema[indexColumn.columnIndex].colId; return TSDB_CODE_SUCCESS; } else { tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + pQueryInfo->order.order = p1->sortOrder; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; // orderby ts query on super table if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - bool found = false; + bool found = false; for (int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { @@ -5693,7 +5761,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } } if (!found && pQueryInfo->pDownstream) { - return invalidOperationMsg(pMsgBuf, msg4); + return invalidOperationMsg(pMsgBuf, msg9); } addPrimaryTsColIntoResult(pQueryInfo, pCmd); } @@ -5701,25 +5769,31 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } else { tVariantListItem *pItem = taosArrayGet(pSqlNode->pSortOrder, 0); if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + pQueryInfo->groupbyExpr.orderIndex = indexColumn.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); pQueryInfo->groupbyExpr.orderType = pItem->sortOrder; } else if (orderByGroupbyCol){ pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = index.columnIndex; + pQueryInfo->order.orderColId = indexColumn.columnIndex; + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } } else { pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } } pItem = taosArrayGet(pSqlNode->pSortOrder, 1); tVariant* pVar2 = &pItem->pVar; SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; - if (getColumnIndexByName(&cname, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&cname, pQueryInfo, &indexColumn, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(pMsgBuf, msg1); } - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg2); + if (indexColumn.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidOperationMsg(pMsgBuf, msg6); } else { tVariantListItem* p1 = taosArrayGet(pSortOrder, 1); pQueryInfo->order.order = p1->sortOrder; @@ -5728,74 +5802,86 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } } else if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { // check order by clause for normal table & temp table - if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&columnName, pQueryInfo, &indexColumn, pMsgBuf) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(pMsgBuf, msg1); } - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pQueryInfo)) { + if (indexColumn.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pQueryInfo)) { bool validOrder = false; SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - validOrder = (pColIndex->colIndex == index.columnIndex); + validOrder = (pColIndex->colIndex == indexColumn.columnIndex); } if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg2); + return invalidOperationMsg(pMsgBuf, msg7); + } + + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId; + pQueryInfo->groupbyExpr.orderIndex = pSchema[indexColumn.columnIndex].colId; pQueryInfo->groupbyExpr.orderType = p1->sortOrder; } if (isTopBottomQuery(pQueryInfo)) { - bool validOrder = false; SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - validOrder = (pColIndex->colIndex == index.columnIndex); - } else { - int32_t topBotIndex = tscGetTopBotQueryExprIndex(pQueryInfo); - assert(topBotIndex >= 1); - /* order of top/bottom query in interval is not valid */ - SExprInfo* pExpr = tscExprGet(pQueryInfo, topBotIndex-1); - assert(pExpr->base.functionId == TSDB_FUNC_TS); - pExpr = tscExprGet(pQueryInfo, topBotIndex); - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg2); + if (pColIndex->colIndex != indexColumn.columnIndex) { + return invalidOperationMsg(pMsgBuf, msg8); } + } else { + int32_t pos = tscGetTopBotQueryExprIndex(pQueryInfo); + assert(pos > 0); + SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); + assert(pExpr->base.functionId == TSDB_FUNC_TS); - validOrder = true; - } + pExpr = tscExprGet(pQueryInfo, pos); - if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg2); + if (pExpr->base.colInfo.colIndex != indexColumn.columnIndex && indexColumn.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidOperationMsg(pMsgBuf, msg5); + } } tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; + pQueryInfo->order.orderColId = pSchema[indexColumn.columnIndex].colId; return TSDB_CODE_SUCCESS; } + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; + pQueryInfo->order.orderColId = pSchema[indexColumn.columnIndex].colId; } else { // handle the temp table order by clause. You can order by any single column in case of the temp table, created by // inner subquery. assert(UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo) && taosArrayGetSize(pSqlNode->pSortOrder) == 1); - if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&columnName, pQueryInfo, &indexColumn, pMsgBuf) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(pMsgBuf, msg1); } + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; + pQueryInfo->order.orderColId = pSchema[indexColumn.columnIndex].colId; } return TSDB_CODE_SUCCESS; @@ -5899,17 +5985,17 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidOperationMsg(pMsg, msg9); } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; - if (getColumnIndexByName(&name, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + if (getColumnIndexByName(&name, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (index.columnIndex < numOfCols) { + if (tsc_index.columnIndex < numOfCols) { return invalidOperationMsg(pMsg, msg10); - } else if (index.columnIndex == numOfCols) { + } else if (tsc_index.columnIndex == numOfCols) { return invalidOperationMsg(pMsg, msg11); } @@ -6478,17 +6564,15 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlN // todo refactor if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query - if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } + if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { + if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } - // for projection query on super table, all queries are subqueries - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { - pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; - } + // for projection query on super table, all queries are subqueries + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { + pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } } @@ -6668,20 +6752,20 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId); int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId); - SColumnIndex index = {.tableIndex = 0, .columnIndex = colIndex}; + SColumnIndex tsc_index = {.tableIndex = 0, .columnIndex = colIndex}; char* name = pTagSchema->name; int16_t type = pTagSchema->type; int16_t bytes = pTagSchema->bytes; - pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, getNewResColId(&pSql->cmd), bytes, true); + pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG, &tsc_index, type, bytes, getNewResColId(&pSql->cmd), bytes, true); pExpr->base.colInfo.flag = TSDB_COL_TAG; // NOTE: tag column does not add to source column list SColumnList ids = {0}; insertResultField(pQueryInfo, (int32_t)size, &ids, bytes, (int8_t)type, name, pExpr); - int32_t relIndex = index.columnIndex; + int32_t relIndex = tsc_index.columnIndex; pExpr->base.colInfo.colIndex = relIndex; SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); @@ -6997,8 +7081,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo int32_t pos = tscGetFirstInvisibleFieldPos(pQueryInfo); - SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - SExprInfo* pExpr = tscExprInsert(pQueryInfo, pos, f, &index, s->type, s->bytes, getNewResColId(pCmd), s->bytes, true); + SColumnIndex tsc_index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; + SExprInfo* pExpr = tscExprInsert(pQueryInfo, pos, f, &tsc_index, s->type, s->bytes, getNewResColId(pCmd), s->bytes, true); memset(pExpr->base.aliasName, 0, sizeof(pExpr->base.aliasName)); tstrncpy(pExpr->base.aliasName, s->name, sizeof(pExpr->base.aliasName)); @@ -7119,6 +7203,11 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* } if (f < 0) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * f - 1); + if (pUdfInfo->funcType == TSDB_UDF_TYPE_SCALAR) { + return invalidOperationMsg(msg, msg1); + } + continue; } @@ -7180,7 +7269,7 @@ int32_t validateFunctionFromUpstream(SQueryInfo* pQueryInfo, char* msg) { SQueryInfo* pUp = taosArrayGetP(pQueryInfo->pUpstream, j); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pUp, 0); bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if ((!isSTable) || groupbyTbname(pUp)) { + if ((!isSTable) || groupbyTbname(pUp) || pUp->interval.interval > 0) { return TSDB_CODE_SUCCESS; } } @@ -7222,20 +7311,20 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq {"client_version()", 16}, {"current_user()", 14}}; - int32_t index = -1; + int32_t tsc_index = -1; if (server_status == true) { - index = 2; + tsc_index = 2; } else { for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { if (strncasecmp(functionsInfo[i].name, pExpr->exprToken.z, functionsInfo[i].len) == 0 && functionsInfo[i].len == pExpr->exprToken.n) { - index = i; + tsc_index = i; break; } } } - switch (index) { + switch (tsc_index) { case 0: pQueryInfo->command = TSDB_SQL_CURRENT_DB;break; case 1: @@ -7254,7 +7343,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq tDataTypes[TSDB_DATA_TYPE_INT].bytes, getNewResColId(pCmd), tDataTypes[TSDB_DATA_TYPE_INT].bytes, false); tSqlExprItem* item = taosArrayGet(pExprList, 0); - const char* name = (item->aliasName != NULL)? item->aliasName:functionsInfo[index].name; + const char* name = (item->aliasName != NULL)? item->aliasName:functionsInfo[tsc_index].name; tstrncpy(pExpr1->base.aliasName, name, tListLen(pExpr1->base.aliasName)); return TSDB_CODE_SUCCESS; @@ -7458,6 +7547,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { const char* msg3 = "tag value too long"; const char* msg4 = "illegal value or data overflow"; const char* msg5 = "tags number not matched"; + const char* msg6 = "create table only from super table is allowed"; SSqlCmd* pCmd = &pSql->cmd; @@ -7498,6 +7588,10 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return code; } + if (!UTIL_TABLE_IS_SUPER_TABLE(pStableMetaInfo)) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + size_t valSize = taosArrayGetSize(pValList); // too long tag values will return invalid sql, not be truncated automatically @@ -7699,12 +7793,12 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - + SRelationInfo* pFromInfo = pInfo->pCreateTableInfo->pSelect->from; if (pFromInfo == NULL || taosArrayGetSize(pFromInfo->list) == 0) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - + SRelElementPair* p1 = taosArrayGet(pFromInfo->list, 0); SStrToken srcToken = {.z = p1->tableName.z, .n = p1->tableName.n, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { @@ -7725,8 +7819,10 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_TSC_INVALID_OPERATION; } + int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); + if (pSqlNode->pWhere != NULL) { // query condition in stream computing - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { + if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -7849,10 +7945,10 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect pParam = taosArrayGet(pSqlExpr->Expr.paramList, 0); SStrToken* pToken = &pParam->pNode->columnName; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - schema = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + getColumnIndexByName(pToken, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); + schema = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, tsc_index.columnIndex); } else { schema = (SSchema) {.colId = PRIMARYKEY_TIMESTAMP_COL_INDEX, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = TSDB_KEYSIZE}; } @@ -7872,15 +7968,15 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect // // if (tSqlExprCompare(pItem->pNode, pSqlExpr) == 0) { // exists, not added it, // -// SColumnIndex index = COLUMN_INDEX_INITIALIZER; +// SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; // int32_t functionId = pSqlExpr->functionId; // if (pSqlExpr->Expr.paramList == NULL) { -// index.columnIndex = 0; -// index.tableIndex = 0; +// tsc_index.columnIndex = 0; +// tsc_index.tableIndex = 0; // } else { // tSqlExprItem* pParamElem = taosArrayGet(pSqlExpr->Expr.paramList, 0); // SStrToken* pToken = &pParamElem->pNode->columnName; -// getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); +// getColumnIndexByName(pToken, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)); // } // // size_t numOfNodeInSel = tscNumOfExprs(pQueryInfo); @@ -7891,7 +7987,7 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect // continue; // } // -// if (pExpr1->base.colInfo.colIndex != index.columnIndex) { +// if (pExpr1->base.colInfo.colIndex != tsc_index.columnIndex) { // continue; // } // @@ -8067,16 +8163,16 @@ int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNode } if (pExpr1->tokenId == TK_ID) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if ((getColumnIndexByName(&pExpr1->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; + if ((getColumnIndexByName(&pExpr1->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (index.columnIndex <= 0 || - index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { + if (tsc_index.columnIndex <= 0 || + tsc_index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -8352,12 +8448,33 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (functionId < 0) { struct SUdfInfo info = {0}; info.name = strndup(t->z, t->n); + info.keep = true; if (pQueryInfo->pUdfInfo == NULL) { pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo)); + } else if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + int32_t usize = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + int32_t exist = 0; + + for (int32_t j = 0; j < usize; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + int32_t len = (int32_t)strlen(pUdfInfo->name); + if (len == t->n && strncasecmp(info.name, pUdfInfo->name, t->n) == 0) { + exist = 1; + break; + } + } + + if (exist) { + continue; + } } info.functionId = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo) * (-1) - 1;; taosArrayPush(pQueryInfo->pUdfInfo, &info); + if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 1) { + code = tscInvalidOperationMsg(pCmd->payload, "only one udf allowed", NULL); + goto _end; + } } } } @@ -8508,8 +8625,8 @@ static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { return meta; } -static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pSql, SQueryInfo* pQueryInfo, char* msgBuf) { - SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, index); +static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t tsc_index, SSqlObj* pSql, SQueryInfo* pQueryInfo, char* msgBuf) { + SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, tsc_index); // union all is not support currently SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); @@ -8587,7 +8704,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf const char* msg4 = "interval query not supported, since the result of sub query not include valid timestamp column"; const char* msg5 = "only tag query not compatible with normal column filter"; const char* msg6 = "not support stddev/percentile/interp in the outer query yet"; - const char* msg7 = "derivative/twa/irate requires timestamp column exists in subquery"; + const char* msg7 = "derivative/twa/rate/irate/diff requires timestamp column exists in subquery"; const char* msg8 = "condition missing for join query"; const char* msg9 = "not support 3 level select"; @@ -8636,6 +8753,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY); + int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); + // parse the group by clause in the first place if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -8671,7 +8790,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { + if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE || + f == TSDB_FUNC_RATE || f == TSDB_FUNC_DIFF) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } } @@ -8679,7 +8799,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf // validate the query filter condition info if (pSqlNode->pWhere != NULL) { - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { + if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } } else { @@ -8724,7 +8844,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf } // parse the having clause in the first place - int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -8772,6 +8891,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t type = isSTable? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_TABLE_QUERY; TSDB_QUERY_SET_TYPE(pQueryInfo->type, type); + int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); + // parse the group by clause in the first place if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -8779,7 +8900,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf pQueryInfo->onlyHasTagCond = true; // set where info if (pSqlNode->pWhere != NULL) { - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { + if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -8790,7 +8911,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf } } - int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); int32_t timeWindowQuery = (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); @@ -8929,7 +9049,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + SColumnIndex tsc_index = COLUMN_INDEX_INITIALIZER; if (pSqlExpr->pLeft != NULL) { int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid); @@ -9001,13 +9121,13 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS } } } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression - int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); + int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &tsc_index, tscGetErrorMsgPayload(pCmd)); if (ret != TSDB_CODE_SUCCESS) { return ret; } - pQueryInfo->curTableIdx = index.tableIndex; - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; + pQueryInfo->curTableIdx = tsc_index.tableIndex; + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, tsc_index.tableIndex)->pTableMeta; int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); *pExpr = calloc(1, sizeof(tExprNode)); @@ -9016,14 +9136,14 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS SSchema* pSchema = NULL; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (tsc_index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pSchema = (*pExpr)->pSchema; strcpy(pSchema->name, TSQL_TBNAME_L); pSchema->type = TSDB_DATA_TYPE_BINARY; pSchema->colId = TSDB_TBNAME_COLUMN_INDEX; pSchema->bytes = -1; } else { - pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); + pSchema = tscGetTableColumnSchema(pTableMeta, tsc_index.columnIndex); *(*pExpr)->pSchema = *pSchema; } @@ -9031,8 +9151,8 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS SColIndex colIndex = {0}; tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name)); colIndex.colId = pSchema->colId; - colIndex.colIndex = index.columnIndex; - colIndex.flag = (index.columnIndex >= numOfColumns)? 1:0; + colIndex.colIndex = tsc_index.columnIndex; + colIndex.flag = (tsc_index.columnIndex >= numOfColumns)? 1:0; taosArrayPush(pCols, &colIndex); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7af5a7df617e4b304736c6cb73b99785af39be25..dd0975de78dfdc231908130c6a2193d0cbc9f2bf 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -860,13 +860,13 @@ static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STab int32_t vgId = -1; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t index = pTableMetaInfo->vgroupIndex; - assert(index >= 0); + int32_t tsc_index = pTableMetaInfo->vgroupIndex; + assert(tsc_index >= 0); SVgroupInfo* pVgroupInfo = NULL; if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { - assert(index < pTableMetaInfo->vgroupList->numOfVgroups); - pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; + assert(tsc_index < pTableMetaInfo->vgroupList->numOfVgroups); + pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[tsc_index]; } else { tscError("0x%"PRIx64" No vgroup info found", pSql->self); @@ -876,7 +876,7 @@ static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STab vgId = pVgroupInfo->vgId; tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); - tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); + tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, tsc_index, pTableMetaInfo->vgroupList->numOfVgroups); } else { vgId = pTableMeta->vgId; @@ -898,11 +898,11 @@ static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STab pQueryMsg->numOfTables = htonl(1); // set the number of tables pMsg += sizeof(STableIdInfo); } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo - int32_t index = pTableMetaInfo->vgroupIndex; + int32_t tsc_index = pTableMetaInfo->vgroupIndex; int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(index >= 0 && index < numOfVgroups); + assert(tsc_index >= 0 && tsc_index < numOfVgroups); - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, tsc_index); // set the vgroup info tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); @@ -912,7 +912,7 @@ static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STab pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, - pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); + pTableIdList->vgInfo.vgId, numOfTables, tsc_index, numOfVgroups); // serialize each table id info for(int32_t i = 0; i < numOfTables; ++i) { @@ -980,7 +980,11 @@ static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, return TSDB_CODE_TSC_INVALID_OPERATION; } - assert(pExpr->resColId < 0); + if (pExpr->resColId >= 0) { + tscError("result column id underflowed: %d", pExpr->resColId); + return TSDB_CODE_TSC_RES_TOO_MANY; + } + SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); SColIndex* pIndex = &pSqlExpr->colInfo; @@ -1237,6 +1241,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // support only one udf if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 1) { + code = tscInvalidOperationMsg(pCmd->payload, "only one udf allowed", NULL); + goto _end; + } + pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); @@ -2616,18 +2625,18 @@ int tscProcessShowRsp(SSqlObj *pSql) { SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - SColumnIndex index = {0}; + SColumnIndex tsc_index = {0}; pSchema = pMetaMsg->schema; uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { - index.columnIndex = i; + tsc_index.columnIndex = i; tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &tsc_index, pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); } @@ -3137,15 +3146,19 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd* pCmd2 = &pSql->rootObj->cmd; pCmd2->pTableMetaMap = tscCleanupTableMetaMap(pCmd2->pTableMetaMap); pCmd2->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - + pSql->rootObj->retryReason = pSql->retryReason; + SSqlObj *tmpSql = pSql->rootObj; + tscFreeSubobj(pSql->rootObj); + tfree(tmpSql->pSubs); + SArray* pNameList = taosArrayInit(1, POINTER_BYTES); SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); char* n = strdup(name); taosArrayPush(pNameList, &n); - code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); + code = getMultiTableMetaFromMnode(tmpSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); taosArrayDestroyEx(pNameList, freeElem); taosArrayDestroyEx(vgroupList, freeElem); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 923b2d74fe9d4f9195241d35ea11d6ac99a9fe8e..eb9e1fb4ad54efefb83a3fdbbd2da8f5a2540bf5 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -417,8 +417,8 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { return; } } else { - int64_t stime = taosTimeTruncate(pStream->stime - 1, &pStream->interval, pStream->precision); - if (stime >= pStream->etime) { + int64_t tsc_stime = taosTimeTruncate(pStream->stime - 1, &pStream->interval, pStream->precision); + if (tsc_stime >= pStream->etime) { tscDebug("0x%"PRIx64" stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql->self, pStream, pStream->stime, pStream->etime); // TODO : How to terminate stream here @@ -498,7 +498,7 @@ static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { return TSDB_CODE_SUCCESS; } -static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) { +static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t tsc_stime) { SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); if (pStream->isProject) { @@ -506,31 +506,31 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in pStream->interval.interval = tsProjectExecInterval; pStream->interval.sliding = tsProjectExecInterval; - if (stime != INT64_MIN) { // first projection start from the latest event timestamp - assert(stime >= pQueryInfo->window.skey); - stime += 1; // exclude the last records from table + if (tsc_stime != INT64_MIN) { // first projection start from the latest event timestamp + assert(tsc_stime >= pQueryInfo->window.skey); + tsc_stime += 1; // exclude the last records from table } else { - stime = pQueryInfo->window.skey; + tsc_stime = pQueryInfo->window.skey; } } else { // timewindow based aggregation stream - if (stime == INT64_MIN) { // no data in meter till now + if (tsc_stime == INT64_MIN) { // no data in meter till now if (pQueryInfo->window.skey != INT64_MIN) { - stime = pQueryInfo->window.skey; + tsc_stime = pQueryInfo->window.skey; } else { - return stime; + return tsc_stime; } - stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); + tsc_stime = taosTimeTruncate(tsc_stime, &pStream->interval, pStream->precision); } else { - int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); - if (newStime != stime) { - tscWarn("0x%"PRIx64" stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql->self, pStream, stime, newStime); - stime = newStime; + int64_t newStime = taosTimeTruncate(tsc_stime, &pStream->interval, pStream->precision); + if (newStime != tsc_stime) { + tscWarn("0x%"PRIx64" stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql->self, pStream, tsc_stime, newStime); + tsc_stime = newStime; } } } - return stime; + return tsc_stime; } static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) { @@ -663,7 +663,7 @@ void cbParseSql(void* param, TAOS_RES* res, int code) { } TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), - int64_t stime, void *param, void (*callback)(void *), void* cqhandle) { + int64_t tsc_stime, void *param, void (*callback)(void *), void* cqhandle) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) return NULL; @@ -692,7 +692,7 @@ TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const c } pStream->ltime = INT64_MIN; - pStream->stime = stime; + pStream->stime = tsc_stime; pStream->fp = fp; pStream->callback = callback; pStream->param = param; @@ -745,8 +745,8 @@ TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const c } TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), - int64_t stime, void *param, void (*callback)(void *)) { - return taos_open_stream_withname(taos, "", sqlstr, fp, stime, param, callback, NULL); + int64_t tsc_stime, void *param, void (*callback)(void *)) { + return taos_open_stream_withname(taos, "", sqlstr, fp, tsc_stime, param, callback, NULL); } void taos_close_stream(TAOS_STREAM *handle) { diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 5e70c814133fd93b7619022a1d564050c3c0502a..5a26397c5a260f8ee527a05bbfc700b53a23951c 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -468,7 +468,7 @@ SSqlObj* recreateSqlObj(SSub* pSub) { } registerSqlObj(pSql); - + pSql->rootObj = pSql; code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { tsem_wait(&pSub->sem); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index af5d21c279d186d33f14768cb69af3c1e887b00a..3a0be5e96d9d5d1518d9f2a1e833e729a0fa8b67 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -380,7 +380,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, STimeWindow * win) { // todo handle failed to create sub query -SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) { +SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t tsc_index) { SJoinSupporter* pSupporter = calloc(1, sizeof(SJoinSupporter)); if (pSupporter == NULL) { return NULL; @@ -388,13 +388,13 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) { pSupporter->pObj = pSql->self; - pSupporter->subqueryIndex = index; + pSupporter->subqueryIndex = tsc_index; SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); memcpy(&pSupporter->interval, &pQueryInfo->interval, sizeof(pSupporter->interval)); pSupporter->limit = pQueryInfo->limit; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, index); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, tsc_index); pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid; assert (pSupporter->uid != 0); @@ -598,7 +598,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { * during the timestamp intersection. */ pSupporter->limit = pQueryInfo->limit; - SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + SColumnIndex tsc_index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0); SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); @@ -610,7 +610,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS; - tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL, getNewResColId(&pNew->cmd)); + tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &tsc_index, s, TSDB_COL_NORMAL, getNewResColId(&pNew->cmd)); tscPrintSelNodeList(pNew, 0); tscFieldInfoUpdateOffset(pQueryInfo); @@ -806,8 +806,8 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; - SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); + SColumnIndex tsc_index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &tsc_index, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); // set the tags value for ts_comp function if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -2272,7 +2272,15 @@ void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { if (row[i] == NULL) { setNull(p + offset, pExpr->base.resType, pExpr->base.resBytes); } else { - memcpy(p + offset, row[i], length[i]); + if(pExpr->base.resType == TSDB_DATA_TYPE_NCHAR){ + int32_t output = 0; + bool ret = taosMbsToUcs4(row[i], length[i], p + offset, pExpr->base.resBytes, &output); + if (!ret) { + tscError("stddev convert tag error:%d", ret); + } + }else{ + memcpy(p + offset, row[i], length[i]); + } } offset += pExpr->base.resBytes; } @@ -2424,7 +2432,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); - int32_t index = 0; + int32_t tsc_index = 0; for(int32_t i = 0; i < numOfExprs; ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); if (pExpr->base.functionId == TSDB_FUNC_TS && pQueryInfo->interval.interval > 0) { @@ -2433,7 +2441,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, tsc_index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); p->base.resColId = pExpr->base.resColId; // update the result column id } else if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); @@ -2442,7 +2450,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SSchema schema = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; tstrncpy(schema.name, pExpr->base.aliasName, tListLen(schema.name)); - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, tsc_index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); p->base.resColId = pExpr->base.resColId; // update the result column id } else if (pExpr->base.functionId == TSDB_FUNC_TAG) { pSup->tagLen += pExpr->base.resBytes; @@ -2455,7 +2463,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { schema = tGetTbnameColumnSchema(); } - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG, getNewResColId(pCmd)); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, tsc_index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG, getNewResColId(pCmd)); p->base.resColId = pExpr->base.resColId; } else if (pExpr->base.functionId == TSDB_FUNC_PRJ) { int32_t num = (int32_t) taosArrayGetSize(pNewQueryInfo->groupbyExpr.columnInfo); @@ -2469,7 +2477,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); //doLimitOutputNormalColOfGroupby - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); + SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, tsc_index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); p->base.numOfParams = 1; p->base.param[0].i64 = 1; p->base.param[0].nType = TSDB_DATA_TYPE_INT; @@ -2511,7 +2519,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { "0x%"PRIx64" first round subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, query to retrieve timestamps, " "numOfExpr:%" PRIzu ", colList:%d, numOfOutputFields:%d, name:%s", pSql->self, pNew->self, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pNewQueryInfo->type, - tscNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); + tscNumOfExprs(pNewQueryInfo), tsc_index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); pSql->pSubs = calloc(1, POINTER_BYTES); if (pSql->pSubs == NULL) { @@ -2881,7 +2889,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO SSqlObj *userSql = pParentSql->rootObj; if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && userSql->retry < userSql->maxRetry) { - if (userSql != pParentSql) { + if (userSql != pParentSql && pParentSql->freeParam != NULL) { (*pParentSql->freeParam)(&pParentSql->param); } @@ -3145,7 +3153,7 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo assert(trsupport->subqueryIndex < pSql->subState.numOfSub); - // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. + // launch subquery for each vnode, so the subquery tsc_index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); pTableMetaInfo->vgroupIndex = trsupport->subqueryIndex; @@ -3628,19 +3636,19 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { char *getArithmeticInputSrc(void *param, const char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *) param; - int32_t index = -1; + int32_t tsc_index = -1; SExprInfo* pExpr = NULL; for (int32_t i = 0; i < pSupport->numOfCols; ++i) { pExpr = taosArrayGetP(pSupport->exprList, i); if (strncmp(name, pExpr->base.aliasName, sizeof(pExpr->base.aliasName) - 1) == 0) { - index = i; + tsc_index = i; break; } } - assert(index >= 0 && index < pSupport->numOfCols); - return pSupport->data[index] + pSupport->offset * pExpr->base.resBytes; + assert(tsc_index >= 0 && tsc_index < pSupport->numOfCols); + return pSupport->data[tsc_index] + pSupport->offset * pExpr->base.resBytes; } TAOS_ROW doSetResultRowData(SSqlObj *pSql) { @@ -3819,7 +3827,7 @@ void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGr pthread_mutex_init(&pQInfo->lock, NULL); tsem_init(&pQInfo->ready, 0, 0); - int32_t index = 0; + int32_t tsc_index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); @@ -3836,7 +3844,7 @@ void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGr STableKeyInfo* info = taosArrayGet(pa, j); window.skey = info->lastKey; - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); + void* buf = (char*) pQInfo->pBuf + tsc_index * sizeof(STableQueryInfo); STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); if (item == NULL) { goto _cleanup; @@ -3847,7 +3855,7 @@ void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGr STableId id = {.tid = 0, .uid = 0}; taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id.tid, sizeof(id.tid), &item, POINTER_BYTES); - index += 1; + tsc_index += 1; } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 8af340030cccee1431a82eb88344642011f2e019..6c08524faa44a7a39e868a705510f2fc2f8275aa 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -45,7 +45,17 @@ int32_t tscNumOfObj = 0; // number of sqlObj in current process. static void *tscCheckDiskUsageTmr; void *tscRpcCache; // cache to keep rpc obj int32_t tscNumOfThreads = 1; // num of rpc threads +#ifdef _TD_POWER_ +char tscLogFileName[12] = "powerlog"; +#elif (_TD_TQ_ == true) +char tscLogFileName[12] = "tqlog"; +#elif (_TD_PRO_ == true) +char tscLogFileName[12] = "prolog"; +#elif (_TD_KH_ == true) +char tscLogFileName[12] = "khclientlog"; +#else char tscLogFileName[12] = "taoslog"; +#endif int tscLogFileNum = 10; static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently @@ -84,27 +94,27 @@ int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncry return 0; } - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "TSC"; - rpcInit.numOfThreads = tscNumOfThreads; - rpcInit.cfp = tscProcessMsgFromServer; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char *)user; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char *)secretEncrypt; + SRpcInit tsc_rpcInit; + memset(&tsc_rpcInit, 0, sizeof(tsc_rpcInit)); + tsc_rpcInit.localPort = 0; + tsc_rpcInit.label = "TSC"; + tsc_rpcInit.numOfThreads = tscNumOfThreads; + tsc_rpcInit.cfp = tscProcessMsgFromServer; + tsc_rpcInit.sessions = tsMaxConnections; + tsc_rpcInit.connType = TAOS_CONN_CLIENT; + tsc_rpcInit.user = (char *)user; + tsc_rpcInit.idleTime = tsShellActivityTimer * 1000; + tsc_rpcInit.ckey = "key"; + tsc_rpcInit.spi = 1; + tsc_rpcInit.secret = (char *)secretEncrypt; SRpcObj rpcObj; memset(&rpcObj, 0, sizeof(rpcObj)); strncpy(rpcObj.key, key, strlen(key)); - rpcObj.pDnodeConn = rpcOpen(&rpcInit); + rpcObj.pDnodeConn = rpcOpen(&tsc_rpcInit); if (rpcObj.pDnodeConn == NULL) { pthread_mutex_unlock(&rpcObjMutex); - tscError("failed to init connection to TDengine"); + tscError("failed to init connection to server"); return -1; } @@ -158,8 +168,7 @@ void taos_init_imp(void) { rpcInit(); scriptEnvPoolInit(); - - tscDebug("starting to initialize TAOS client ..."); + tscDebug("starting to initialize client ..."); tscDebug("Local End Point is:%s", tsLocalEp); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6a3ceef2c242f6f8ca7781db8694580f180d3358..b47faa32d9100ff33139137b0ba05e2594469a9f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -166,7 +166,7 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) { continue; } - if (functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_TID_TAG) { + if (functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_TID_TAG && functId != TSDB_FUNC_BLKINFO) { return false; } } @@ -919,23 +919,31 @@ SSDataBlock* doGetDataBlock(void* param, bool* newgroup) { pBlock->info.rows = pRes->numOfRows; if (pRes->numOfRows != 0) { doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo, pInput->numOfFilterCols); - *newgroup = false; - return pBlock; + if (pBlock->info.rows > 0) { + *newgroup = false; + return pBlock; + } } - // No data block exists. So retrieve and transfer it into to SSDataBlock - TAOS_ROW pRow = NULL; - taos_fetch_block(pSql, &pRow); + SSDataBlock* result = NULL; + do { + // No data block exists. So retrieve and transfer it into to SSDataBlock + TAOS_ROW pRow = NULL; + taos_fetch_block(pSql, &pRow); - if (pRes->numOfRows == 0) { - pOperator->status = OP_EXEC_DONE; - return NULL; - } + if (pRes->numOfRows == 0) { + pOperator->status = OP_EXEC_DONE; + result = NULL; + break; + } - pBlock->info.rows = pRes->numOfRows; - doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo, pInput->numOfFilterCols); - *newgroup = false; - return pBlock; + pBlock->info.rows = pRes->numOfRows; + doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo, pInput->numOfFilterCols); + *newgroup = false; + result = pBlock; + } while (result->info.rows == 0); + + return result; } static void fetchNextBlockIfCompleted(SOperatorInfo* pOperator, bool* newgroup) { @@ -1246,6 +1254,28 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue .pGroupList = taosArrayInit(1, POINTER_BYTES), }; + SUdfInfo* pUdfInfo = NULL; + + size_t size = tscNumOfExprs(px); + for (int32_t j = 0; j < size; ++j) { + SExprInfo* pExprInfo = tscExprGet(px, j); + + int32_t functionId = pExprInfo->base.functionId; + if (functionId < 0) { + if (pUdfInfo) { + pSql->res.code = tscInvalidOperationMsg(pSql->cmd.payload, "only one udf allowed", NULL); + return; + } + + pUdfInfo = taosArrayGet(px->pUdfInfo, -1 * functionId - 1); + int32_t code = initUdfInfo(pUdfInfo); + if (code != TSDB_CODE_SUCCESS) { + pSql->res.code = code; + return; + } + } + } + tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; @@ -1315,6 +1345,9 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute the main query while all nest queries are ready", pSql->self, pSql->self); px->pQInfo = createQInfoFromQueryNode(px, &tableGroupInfo, pSourceOperator, NULL, NULL, MASTER_SCAN, pSql->self); + px->pQInfo->runtimeEnv.udfIsCopy = true; + px->pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; + tfree(pColumnInfo); tfree(schema); @@ -1772,6 +1805,32 @@ int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOff return TSDB_CODE_TSC_OUT_OF_MEMORY; } + int32_t code = tscCreateDataBlockData(dataBuf, defaultSize, rowSize, startOffset); + if (code != TSDB_CODE_SUCCESS) { + tfree(dataBuf); + return code; + } + + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); + + SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; + SSchema* pSchema = tscGetTableSchema(dataBuf->pTableMeta); + tscSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); + + dataBuf->vgId = dataBuf->pTableMeta->vgId; + + tNameAssign(&dataBuf->tableName, name); + + assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + + *dataBlocks = dataBuf; + return TSDB_CODE_SUCCESS; +} + +int32_t tscCreateDataBlockData(STableDataBlocks* dataBuf, size_t defaultSize, int32_t rowSize, int32_t startOffset) { + assert(dataBuf != NULL); + dataBuf->nAllocSize = (uint32_t)defaultSize; dataBuf->headerSize = startOffset; @@ -1784,30 +1843,16 @@ int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOff dataBuf->pData = malloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { tscError("failed to allocated memory, reason:%s", strerror(errno)); - tfree(dataBuf); return TSDB_CODE_TSC_OUT_OF_MEMORY; } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - //Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); - - SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; - SSchema* pSchema = tscGetTableSchema(dataBuf->pTableMeta); - tscSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); - dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; dataBuf->rowSize = rowSize; dataBuf->size = startOffset; dataBuf->tsSource = -1; - dataBuf->vgId = dataBuf->pTableMeta->vgId; - - tNameAssign(&dataBuf->tableName, name); - - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); - *dataBlocks = dataBuf; return TSDB_CODE_SUCCESS; } @@ -1898,18 +1943,11 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI } } else { for (int32_t i = 0; i < numOfRows; ++i) { - char* payload = (blkKeyTuple + i)->payloadAddr; - if (isNeedConvertRow(payload)) { - convertSMemRow(pDataBlock, payload, pTableDataBlock); - TDRowTLenT rowTLen = memRowTLen(pDataBlock); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } else { - TDRowTLenT rowTLen = memRowTLen(payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } + char* payload = (blkKeyTuple + i)->payloadAddr; + TDRowTLenT rowTLen = memRowTLen(payload); + memcpy(pDataBlock, payload, rowTLen); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; } } @@ -2087,10 +2125,10 @@ void tscCloseTscObj(void *param) { } bool tscIsInsertData(char* sqlstr) { - int32_t index = 0; + int32_t tsc_index = 0; do { - SStrToken t0 = tStrGetToken(sqlstr, &index, false); + SStrToken t0 = tStrGetToken(sqlstr, &tsc_index, false); if (t0.type != TK_LP) { return t0.type == TK_INSERT || t0.type == TK_IMPORT; } @@ -2157,12 +2195,12 @@ SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { return taosArrayPush(pFieldInfo->internalField, &info); } -SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { +SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t tsc_index, TAOS_FIELD* field) { pFieldInfo->numOfOutput++; struct SInternalField info = { .pExpr = NULL, .visible = true }; info.field = *field; - return taosArrayInsert(pFieldInfo->internalField, index, &info); + return taosArrayInsert(pFieldInfo->internalField, tsc_index, &info); } void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { @@ -2177,18 +2215,18 @@ void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { } } -SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index) { - assert(index < pFieldInfo->numOfOutput); - return TARRAY_GET_ELEM(pFieldInfo->internalField, index); +SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t tsc_index) { + assert(tsc_index < pFieldInfo->numOfOutput); + return TARRAY_GET_ELEM(pFieldInfo->internalField, tsc_index); } -TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { - assert(index < pFieldInfo->numOfOutput); - return &((SInternalField*)TARRAY_GET_ELEM(pFieldInfo->internalField, index))->field; +TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t tsc_index) { + assert(tsc_index < pFieldInfo->numOfOutput); + return &((SInternalField*)TARRAY_GET_ELEM(pFieldInfo->internalField, tsc_index))->field; } -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, index); +int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t tsc_index) { + SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, tsc_index); assert(pInfo != NULL && pInfo->pExpr->pExpr == NULL); return pInfo->pExpr->base.offset; @@ -2357,7 +2395,7 @@ SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SCo SSqlExpr* p = &pExpr->base; p->functionId = functionId; - // set the correct columnIndex index + // set the correct columnIndex tsc_index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { SSchema* s = tGetTbnameColumnSchema(); p->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; @@ -2404,16 +2442,16 @@ SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SCo return pExpr; } -SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t tsc_index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { int32_t num = (int32_t)taosArrayGetSize(pQueryInfo->exprList); - if (index == num) { + if (tsc_index == num) { return tscExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SExprInfo* pExpr = tscExprCreate(pTableMetaInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); - taosArrayInsert(pQueryInfo->exprList, index, &pExpr); + taosArrayInsert(pQueryInfo->exprList, tsc_index, &pExpr); return pExpr; } @@ -2425,10 +2463,10 @@ SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnInde return pExpr; } -SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, +SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t tsc_index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SExprInfo* pExpr = tscExprGet(pQueryInfo, index); + SExprInfo* pExpr = tscExprGet(pQueryInfo, tsc_index); if (pExpr == NULL) { return NULL; } @@ -2445,8 +2483,8 @@ SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t function return pExpr; } -bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) { - if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) { +bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tsc_index) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[tsc_index])) { return false; } @@ -2477,8 +2515,8 @@ void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t byt assert(pExpr->numOfParams <= 3); } -SExprInfo* tscExprGet(SQueryInfo* pQueryInfo, int32_t index) { - return taosArrayGetP(pQueryInfo->exprList, index); +SExprInfo* tscExprGet(SQueryInfo* pQueryInfo, int32_t tsc_index) { + return taosArrayGetP(pQueryInfo->exprList, tsc_index); } /* @@ -3000,8 +3038,8 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int16_t index = pExpr->base.colInfo.colIndex; - pColInfo[i].type = (index != -1) ? pTagSchema[index].type : TSDB_DATA_TYPE_BINARY; + int16_t tsc_index = pExpr->base.colInfo.colIndex; + pColInfo[i].type = (tsc_index != -1) ? pTagSchema[tsc_index].type : TSDB_DATA_TYPE_BINARY; } else { pColInfo[i].type = pSchema[pExpr->base.colInfo.colIndex].type; } @@ -3081,7 +3119,7 @@ SQueryInfo* tscGetQueryInfoS(SSqlCmd* pCmd) { return pQueryInfo; } -STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { +STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* tsc_index) { int32_t k = -1; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { @@ -3091,8 +3129,8 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i } } - if (index != NULL) { - *index = k; + if (tsc_index != NULL) { + *tsc_index = k; } assert(k != -1); @@ -3210,6 +3248,7 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { pQueryInfo->sessionWindow = pSrc->sessionWindow; pQueryInfo->pTableMetaInfo = NULL; pQueryInfo->multigroupResult = pSrc->multigroupResult; + pQueryInfo->stateWindow = pSrc->stateWindow; pQueryInfo->bufLen = pSrc->bufLen; pQueryInfo->orderProjectQuery = pSrc->orderProjectQuery; @@ -3302,15 +3341,15 @@ void tscFreeVgroupTableInfo(SArray* pVgroupTables) { taosArrayDestroy(pVgroupTables); } -void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { - assert(pVgroupTable != NULL && index >= 0); +void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t tsc_index) { + assert(pVgroupTable != NULL && tsc_index >= 0); size_t size = taosArrayGetSize(pVgroupTable); - assert(size > index); + assert(size > tsc_index); - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index); + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, tsc_index); taosArrayDestroy(pInfo->itemList); - taosArrayRemove(pVgroupTable, index); + taosArrayRemove(pVgroupTable, tsc_index); } void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { @@ -3780,15 +3819,15 @@ static void tscSubqueryRetrieveCallback(void* param, TAOS_RES* tres, int code) { SSqlObj* pParentSql = ps->pParentSql; SSqlObj* pSql = tres; - int32_t index = ps->subqueryIndex; - bool ret = subAndCheckDone(pSql, pParentSql, index); + int32_t tsc_index = ps->subqueryIndex; + bool ret = subAndCheckDone(pSql, pParentSql, tsc_index); // TODO refactor tfree(ps); pSql->param = NULL; if (!ret) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, tsc_index); return; } @@ -3809,13 +3848,13 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { if (pSql->res.code != TSDB_CODE_SUCCESS) { SSqlObj* pParentSql = ps->pParentSql; - int32_t index = ps->subqueryIndex; - bool ret = subAndCheckDone(pSql, pParentSql, index); + int32_t tsc_index = ps->subqueryIndex; + bool ret = subAndCheckDone(pSql, pParentSql, tsc_index); tscFreeRetrieveSup(&pSql->param); if (!ret) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, tsc_index); return; } @@ -4560,7 +4599,7 @@ int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaI } } - pse->colInfo.flag = TSDB_COL_NORMAL; + pse->colInfo.flag = pSource->base.colInfo.flag; //TSDB_COL_NORMAL; pse->resType = pSource->base.resType; pse->resBytes = pSource->base.resBytes; strncpy(pse->colInfo.name, pSource->base.aliasName, tListLen(pse->colInfo.name)); @@ -4575,9 +4614,14 @@ int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaI functionId = TSDB_FUNC_STDDEV; } + SUdfInfo* pUdfInfo = NULL; + if (functionId < 0) { + pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); + } + int32_t inter = 0; getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType, - &pse->resBytes, &inter, 0, false, NULL); + &pse->resBytes, &inter, 0, false, pUdfInfo); pse->colType = pse->resType; pse->colBytes = pse->resBytes; diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index c18597c7db21ed3f36553d5ba531347afbb639ee..c93945b09ea61cafbca30120950c4d3ee1bcf645 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -448,13 +448,14 @@ typedef struct { #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) #define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) #define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) +#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t))) #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) #define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) +#define kvRowTKey(r) (*(TKEY *)(kvRowKeys(r))) #define kvRowKey(r) tdGetKey(kvRowTKey(r)) #define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) @@ -608,22 +609,17 @@ typedef void *SMemRow; #define SMEM_ROW_DATA 0x0U // SDataRow #define SMEM_ROW_KV 0x01U // SKVRow -#define SMEM_ROW_CONVERT 0x80U // SMemRow convert flag -#define KVRatioKV (0.2f) // all bool -#define KVRatioPredict (0.4f) -#define KVRatioData (0.75f) // all bigint #define KVRatioConvert (0.9f) #define memRowType(r) ((*(uint8_t *)(r)) & 0x01) #define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory -#define memRowSetConvert(r) ((*(uint8_t *)(r)) = (((*(uint8_t *)(r)) & 0x7F) | SMEM_ROW_CONVERT)) // highest bit #define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) #define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) #define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) #define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) -#define isNeedConvertRow(r) (((*(uint8_t *)(r)) & 0x80) == SMEM_ROW_CONVERT) +#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag #define memRowKvBody(r) \ @@ -652,7 +648,7 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) { #define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) #define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version #define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) -#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) +#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r))) #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) #define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 8a6cc9ddb5a4425e3d7bd10bb7dd5dda0712b0c3..ffa89b8c39bea339e7916cc027232e88bfbaa44b 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -45,7 +45,7 @@ extern int32_t tsDnodeId; // common extern int tsRpcTimer; extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled +extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; @@ -56,18 +56,19 @@ extern float tsRatioOfQueryCores; extern int8_t tsDaylight; extern char tsTimezone[]; extern char tsLocale[]; -extern char tsCharset[]; // default encode string +extern char tsCharset[]; // default encode string extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern int32_t tsMaxNumOfDistinctResults; extern char tsTempDir[]; -//query buffer management -extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing -extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing -extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked +// query buffer management +extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing +extern int64_t + tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing +extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked -extern int8_t tsKeepOriginalColumnName; +extern int8_t tsKeepOriginalColumnName; // client extern int32_t tsMaxSQLStringLen; @@ -126,6 +127,7 @@ extern int8_t tsHttpEnableCompress; extern int8_t tsHttpEnableRecordSql; extern int8_t tsTelegrafUseFieldNum; extern int8_t tsHttpDbNameMandatory; +extern int32_t tsHttpKeepAlive; // mqtt extern int8_t tsEnableMqttModule; @@ -162,22 +164,22 @@ extern int64_t tsTickPerDay[3]; extern int32_t tsTopicBianryLen; // system info -extern char tsOsName[]; -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; -extern int32_t tsTotalMemoryMB; +extern char tsOsName[]; +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern float tsTotalLogDirGB; +extern float tsTotalTmpDirGB; +extern float tsTotalDataDirGB; +extern float tsAvailLogDirGB; +extern float tsAvailTmpDirectorySpace; +extern float tsAvailDataDirGB; +extern float tsUsedDataDirGB; +extern float tsMinimalLogDirGB; +extern float tsReservedTmpDirectorySpace; +extern float tsMinimalDataDirGB; +extern int32_t tsTotalMemoryMB; extern uint32_t tsVersion; // build info @@ -188,26 +190,26 @@ extern char gitinfoOfInternal[]; extern char buildinfo[]; // log -extern int8_t tsAsyncLog; -extern int32_t tsNumOfLogLines; -extern int32_t tsLogKeepDays; -extern int32_t dDebugFlag; -extern int32_t vDebugFlag; -extern int32_t mDebugFlag; +extern int8_t tsAsyncLog; +extern int32_t tsNumOfLogLines; +extern int32_t tsLogKeepDays; +extern int32_t dDebugFlag; +extern int32_t vDebugFlag; +extern int32_t mDebugFlag; extern uint32_t cDebugFlag; -extern int32_t jniDebugFlag; -extern int32_t tmrDebugFlag; -extern int32_t sdbDebugFlag; -extern int32_t httpDebugFlag; -extern int32_t mqttDebugFlag; -extern int32_t monDebugFlag; -extern int32_t uDebugFlag; -extern int32_t rpcDebugFlag; -extern int32_t odbcDebugFlag; +extern int32_t jniDebugFlag; +extern int32_t tmrDebugFlag; +extern int32_t sdbDebugFlag; +extern int32_t httpDebugFlag; +extern int32_t mqttDebugFlag; +extern int32_t monDebugFlag; +extern int32_t uDebugFlag; +extern int32_t rpcDebugFlag; +extern int32_t odbcDebugFlag; extern uint32_t qDebugFlag; -extern int32_t wDebugFlag; -extern int32_t cqDebugFlag; -extern int32_t debugFlag; +extern int32_t wDebugFlag; +extern int32_t cqDebugFlag; +extern int32_t debugFlag; extern int8_t tsClientMerge; @@ -218,9 +220,11 @@ extern double fPrecision; extern double dPrecision; extern uint32_t maxRange; extern uint32_t curRange; -extern char Compressor[]; +extern char Compressor[]; #endif +// long query +extern int8_t tsDeadLockKillQuery; typedef struct { char dir[TSDB_FILENAME_LEN]; diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 21b7fd8223d59f57ed3c00eed019048a901ddf96..0889fc46c35a66e863f4c21c1e08acca7b717f40 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -41,6 +41,8 @@ bool tVariantIsValid(tVariant *pVar); void tVariantCreate(tVariant *pVar, SStrToken *token); +void tVariantCreateExt(tVariant *pVar, SStrToken *token, int32_t optrType); + void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type); void tVariantDestroy(tVariant *pV); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 2578f33246bc416e93ada6e442ae27741f3f4253..cba61b9317bba4a9b41d51c3a95cfac52e0b0281 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -253,9 +253,10 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo } if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1; - if (numOfRows > 0) { + + if (((rowOffset == 0) && (numOfRows > 0)) || ((rowOffset == -1) && (numOfRows >= 0))) { // Find the first not null value, fill all previouse values as NULL - dataColSetNEleNull(pCol, numOfRows); + dataColSetNEleNull(pCol, numOfRows - rowOffset); } } @@ -303,14 +304,14 @@ bool isNEleNull(SDataCol *pCol, int nEle) { return true; } -static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { +static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int utl_index) { if (IS_VAR_DATA_TYPE(pCol->type)) { - pCol->dataOff[index] = pCol->len; + pCol->dataOff[utl_index] = pCol->len; char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); setVardataNull(ptr, pCol->type); pCol->len += varDataTLen(ptr); } else { - setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); + setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * utl_index), pCol->type, pCol->bytes); pCol->len += TYPE_BYTES[pCol->type]; } } @@ -463,9 +464,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols int rcol = 0; int dcol = 0; - while (dcol < pCols->numOfCols) { - bool setCol = 0; SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, rowOffset); @@ -476,14 +475,22 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols STColumn *pRowCol = schemaColAt(pSchema, rcol); if (pRowCol->colId == pDataCol->colId) { void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE); - if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints, rowOffset); + if (rowOffset == 0) { + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints, rowOffset); + } else if (rowOffset == -1) { + // for update 2 + if (!isNull(value, pDataCol->type)) { + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints, rowOffset); + } + } else { + ASSERT(0); + } dcol++; rcol++; } else if (pRowCol->colId < pDataCol->colId) { rcol++; } else { - if(forceSetNull || setCol) { + if(forceSetNull) { dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, rowOffset); } dcol++; @@ -501,7 +508,6 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo int nRowCols = kvRowNCols(row); while (dcol < pCols->numOfCols) { - bool setCol = 0; SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, rowOffset); @@ -513,14 +519,22 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo if (colIdx->colId == pDataCol->colId) { void *value = tdGetKvRowDataOfCol(row, colIdx->offset); - if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints, rowOffset); + if (rowOffset == 0) { + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints, rowOffset); + } else if (rowOffset == -1) { + // for update 2 + if (!isNull(value, pDataCol->type)) { + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints, rowOffset); + } + } else { + ASSERT(0); + } ++dcol; ++rcol; } else if (colIdx->colId < pDataCol->colId) { ++rcol; } else { - if (forceSetNull || setCol) { + if (forceSetNull) { dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints, rowOffset); } ++dcol; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index f1a7e4f7fd12a85ed1b8b503db35fba7211fc788..6d473aeec2a14d3dad16f2e7cec80fc7d490c467 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -14,18 +14,18 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" +#include "monitor.h" #include "os.h" #include "taosdef.h" #include "taoserror.h" -#include "tulog.h" +#include "tcompare.h" #include "tconfig.h" -#include "tglobal.h" -#include "monitor.h" -#include "tsocket.h" -#include "tutil.h" #include "tlocale.h" +#include "tsocket.h" #include "ttimezone.h" -#include "tcompare.h" +#include "tulog.h" +#include "tutil.h" // TSDB bool tsdbForceKeepFile = false; @@ -51,16 +51,16 @@ char tsEmail[TSDB_FQDN_LEN] = {0}; int32_t tsDnodeId = 0; // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 0; //disable this, means query, show command use udp protocol as default -int32_t tsMaxShellConns = 50000; +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; +int32_t tsRpcForceTcp = 0; // disable this, means query, show command use udp protocol as default +int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; -int32_t tsShellActivityTimer = 3; // second +int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; -int8_t tsDaylight = 0; +int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string @@ -75,7 +75,7 @@ int32_t tsMaxBinaryDisplayWidth = 30; * -1: all data are not compressed * other values: if the message payload size is greater than the tsCompressMsgSize, the message will be compressed. */ -int32_t tsCompressMsgSize = -1; +int32_t tsCompressMsgSize = 512 * 1024; // client int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; @@ -90,7 +90,7 @@ int32_t tsMaxNumOfOrderedResults = 1000000; int32_t tsMinSlidingTime = 10; // the maxinum number of distict query result -int32_t tsMaxNumOfDistinctResults = 1000 * 10000; +int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 us for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; @@ -102,7 +102,7 @@ int32_t tsMaxStreamComputDelay = 20000; int32_t tsStreamCompStartDelay = 10000; // the stream computing delay time after executing failed, change accordingly -int32_t tsRetryStreamCompDelay = 10*1000; +int32_t tsRetryStreamCompDelay = 10 * 1000; // The delayed computing ration. 10% of the whole computing time window by default. float tsStreamComputDelayRatio = 0.1f; @@ -121,26 +121,26 @@ int64_t tsQueryBufferSizeBytes = -1; int32_t tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name -int8_t tsKeepOriginalColumnName = 0; +int8_t tsKeepOriginalColumnName = 0; // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS; -int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; -int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; +int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; +int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; -int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds +int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; -int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; -int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; -int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; -int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; -int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int16_t tsPartitons = TSDB_DEFAULT_DB_PARTITON_OPTION; -int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; -int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; -int32_t tsMaxVgroupsPerDb = 0; +int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; +int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; +int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; +int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; +int16_t tsPartitons = TSDB_DEFAULT_DB_PARTITON_OPTION; +int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; +int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; +int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; @@ -148,7 +148,7 @@ int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; // balance int8_t tsEnableBalance = 1; int8_t tsAlternativeRole = 0; -int32_t tsBalanceInterval = 300; // seconds +int32_t tsBalanceInterval = 300; // seconds int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days int32_t tsMnodeEqualVnodeNum = 4; int8_t tsEnableFlowCtrl = 1; @@ -166,15 +166,16 @@ int8_t tsHttpEnableCompress = 1; int8_t tsHttpEnableRecordSql = 0; int8_t tsTelegrafUseFieldNum = 0; int8_t tsHttpDbNameMandatory = 0; +int32_t tsHttpKeepAlive = 30000; // mqtt int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default -char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org"; -char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883"; -char tsMqttUser[TSDB_MQTT_USER_LEN] = {0}; -char tsMqttPass[TSDB_MQTT_PASS_LEN] = {0}; -char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber"; -char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // # +char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org"; +char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883"; +char tsMqttUser[TSDB_MQTT_USER_LEN] = {0}; +char tsMqttPass[TSDB_MQTT_PASS_LEN] = {0}; +char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber"; +char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // # // monitor int8_t tsEnableMonitorModule = 1; @@ -183,7 +184,7 @@ char tsInternalPass[] = "secretkey"; int32_t tsMonitorInterval = 30; // seconds // stream -int8_t tsEnableStream = 1; +int8_t tsEnableStream = 1; // internal int8_t tsCompactMnodeWal = 0; @@ -199,7 +200,7 @@ char tsDataDir[PATH_MAX] = {0}; char tsScriptDir[PATH_MAX] = {0}; char tsTempDir[PATH_MAX] = "/tmp/"; -int32_t tsDiskCfgNum = 0; +int32_t tsDiskCfgNum = 0; int32_t tsTopicBianryLen = 16000; #ifndef _STORAGE @@ -217,42 +218,42 @@ SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; // system info -char tsOsName[10] = "Linux"; -int64_t tsPageSize; -int64_t tsOpenMax; -int64_t tsStreamMax; -int32_t tsNumOfCores = 1; -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; -int32_t tsTotalMemoryMB = 0; +char tsOsName[10] = "Linux"; +int64_t tsPageSize; +int64_t tsOpenMax; +int64_t tsStreamMax; +int32_t tsNumOfCores = 1; +float tsTotalTmpDirGB = 0; +float tsTotalDataDirGB = 0; +float tsAvailTmpDirectorySpace = 0; +float tsAvailDataDirGB = 0; +float tsUsedDataDirGB = 0; +float tsReservedTmpDirectorySpace = 1.0f; +float tsMinimalDataDirGB = 2.0f; +int32_t tsTotalMemoryMB = 0; uint32_t tsVersion = 0; // log -int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 131; -int32_t sdbDebugFlag = 131; -int32_t dDebugFlag = 135; -int32_t vDebugFlag = 135; +int32_t tsNumOfLogLines = 10000000; +int32_t mDebugFlag = 131; +int32_t sdbDebugFlag = 131; +int32_t dDebugFlag = 135; +int32_t vDebugFlag = 135; uint32_t cDebugFlag = 131; -int32_t jniDebugFlag = 131; -int32_t odbcDebugFlag = 131; -int32_t httpDebugFlag = 131; -int32_t mqttDebugFlag = 131; -int32_t monDebugFlag = 131; +int32_t jniDebugFlag = 131; +int32_t odbcDebugFlag = 131; +int32_t httpDebugFlag = 131; +int32_t mqttDebugFlag = 131; +int32_t monDebugFlag = 131; uint32_t qDebugFlag = 131; -int32_t rpcDebugFlag = 131; -int32_t uDebugFlag = 131; -int32_t debugFlag = 0; -int32_t sDebugFlag = 135; -int32_t wDebugFlag = 135; -int32_t tsdbDebugFlag = 131; -int32_t cqDebugFlag = 131; -int32_t fsDebugFlag = 135; +int32_t rpcDebugFlag = 131; +int32_t uDebugFlag = 131; +int32_t debugFlag = 0; +int32_t sDebugFlag = 135; +int32_t wDebugFlag = 135; +int32_t tsdbDebugFlag = 131; +int32_t cqDebugFlag = 131; +int32_t fsDebugFlag = 135; int8_t tsClientMerge = 0; @@ -260,15 +261,19 @@ int8_t tsClientMerge = 0; // // lossy compress 6 // -char lossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty can close lossy compress. -// below option can take effect when tsLossyColumns not empty -double fPrecision = 1E-8; // float column precision -double dPrecision = 1E-16; // double column precision -uint32_t maxRange = 500; // max range -uint32_t curRange = 100; // range -char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR +char lossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty + // can close lossy compress. +// below option can take effect when tsLossyColumns not empty +double fPrecision = 1E-8; // float column precision +double dPrecision = 1E-16; // double column precision +uint32_t maxRange = 500; // max range +uint32_t curRange = 100; // range +char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR #endif +// long query death-lock +int8_t tsDeadLockKillQuery = 1; + int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; void (*monExecuteSQLFp)(char *sql) = NULL; @@ -278,7 +283,7 @@ char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { - if (debugFlag != 0) { + if (debugFlag != 0) { mDebugFlag = debugFlag; sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; @@ -289,7 +294,7 @@ void taosSetAllDebugFlag() { httpDebugFlag = debugFlag; mqttDebugFlag = debugFlag; monDebugFlag = debugFlag; - qDebugFlag = debugFlag; + qDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; sDebugFlag = debugFlag; @@ -301,12 +306,13 @@ void taosSetAllDebugFlag() { } bool taosCfgDynamicOptions(char *msg) { - char *option, *value; - int32_t olen, vlen; - int32_t vint = 0; + char *option, *value; + int32_t olen, vlen; + int32_t vint = 0; paGetToken(msg, &option, &olen); - if (olen == 0) return false;; + if (olen == 0) return false; + ; paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) @@ -319,9 +325,9 @@ bool taosCfgDynamicOptions(char *msg) { for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; - //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; + // if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; - + int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; @@ -350,7 +356,7 @@ bool taosCfgDynamicOptions(char *msg) { return true; } if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); + taosSetAllDebugFlag(); } return true; } @@ -373,10 +379,10 @@ bool taosCfgDynamicOptions(char *msg) { return false; } -void taosAddDataDir(int index, char *v1, int level, int primary) { - tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); - tsDiskCfg[index].level = level; - tsDiskCfg[index].primary = primary; +void taosAddDataDir(int utl_index, char *v1, int level, int primary) { + tstrncpy(tsDiskCfg[utl_index].dir, v1, TSDB_FILENAME_LEN); + tsDiskCfg[utl_index].level = level; + tsDiskCfg[utl_index].primary = primary; uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); } @@ -407,7 +413,7 @@ static void taosCheckDataDirCfg() { } static int32_t taosCheckTmpDir(void) { - if (strlen(tsTempDir) <= 0){ + if (strlen(tsTempDir) <= 0) { uError("tempDir is not set"); return -1; } @@ -428,7 +434,7 @@ static void doInitGlobalConfig(void) { srand(taosSafeRand()); SGlobalCfg cfg = {0}; - + // ip address cfg.option = "firstEp"; cfg.ptr = tsFirst; @@ -557,12 +563,12 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsMaxNumOfDistinctResults; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10*10000; - cfg.maxValue = 10000*10000; + cfg.minValue = 10 * 10000; + cfg.maxValue = 10000 * 10000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - + cfg.option = "numOfMnodes"; cfg.ptr = &tsNumOfMnodes; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1149,7 +1155,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - // module configs + // module configs cfg.option = "flowctrl"; cfg.ptr = &tsEnableFlowCtrl; cfg.valType = TAOS_CFG_VTYPE_INT8; @@ -1273,13 +1279,24 @@ static void doInitGlobalConfig(void) { cfg.option = "httpDbNameMandatory"; cfg.ptr = &tsHttpDbNameMandatory; cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + // pContext in cache + cfg.option = "httpKeepAlive"; + cfg.ptr = &tsHttpKeepAlive; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 3000; + cfg.maxValue = 3600000; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + // debug flag cfg.option = "numOfLogLines"; cfg.ptr = &tsNumOfLogLines; @@ -1361,7 +1378,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "sdbDebugFlag"; cfg.ptr = &sdbDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1593,6 +1609,17 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + // enable kill long query + cfg.option = "deadLockKillQuery"; + cfg.ptr = &tsDeadLockKillQuery; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + #ifdef TD_TSZ // lossy compress cfg.option = "lossyColumns"; @@ -1648,21 +1675,18 @@ static void doInitGlobalConfig(void) { #else assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif - } -void taosInitGlobalCfg() { - pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); -} +void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } int32_t taosCheckGlobalCfg() { - char fqdn[TSDB_FQDN_LEN]; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { taosSetAllDebugFlag(); } - + if (tsLocalFqdn[0] == 0) { taosGetFqdn(tsLocalFqdn); } @@ -1689,7 +1713,7 @@ int32_t taosCheckGlobalCfg() { if (taosCheckTmpDir()) { return -1; } - + taosGetSystemInfo(); tsSetLocale(); @@ -1711,8 +1735,8 @@ int32_t taosCheckGlobalCfg() { } if (tsMaxTablePerVnode < tsMinTablePerVnode) { - uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", - tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode); + uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", tsMaxTablePerVnode, + tsMinTablePerVnode, tsMinTablePerVnode); tsMaxTablePerVnode = tsMinTablePerVnode; } @@ -1734,7 +1758,7 @@ int32_t taosCheckGlobalCfg() { } tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035] - tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE; // udp/tcp + tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE; // udp/tcp tsSyncPort = tsServerPort + TSDB_PORT_SYNC; tsHttpPort = tsServerPort + TSDB_PORT_HTTP; @@ -1754,17 +1778,17 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { strcpy(fqdn, ep); char *temp = strchr(fqdn, ':'); - if (temp) { + if (temp) { *temp = 0; - *port = atoi(temp+1); - } - + *port = atoi(temp + 1); + } + if (*port == 0) { *port = tsServerPort; return -1; } - return 0; + return 0; } /* diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 5da48b2e9ac9e8bdaf5158ae780379c913275780..8dcecf650bcf60f4388cc49a40aa4dd365c71ff7 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -338,29 +338,29 @@ void tNameAssign(SName* dst, const SName* src) { memcpy(dst, src, sizeof(SName)); } -int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken) { - assert(dst != NULL && dbToken != NULL && acct != NULL); +int32_t tNameSetDbName(SName* dst, const char* utl_acct, SStrToken* dbToken) { + assert(dst != NULL && dbToken != NULL && utl_acct != NULL); // too long account id or too long db name - if (strlen(acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) { + if (strlen(utl_acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) { return -1; } dst->type = TSDB_DB_NAME_T; - tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + tstrncpy(dst->acctId, utl_acct, tListLen(dst->acctId)); tstrncpy(dst->dbname, dbToken->z, dbToken->n + 1); return 0; } -int32_t tNameSetAcctId(SName* dst, const char* acct) { - assert(dst != NULL && acct != NULL); +int32_t tNameSetAcctId(SName* dst, const char* utl_acct) { + assert(dst != NULL && utl_acct != NULL); // too long account id or too long db name - if (strlen(acct) >= tListLen(dst->acctId)) { + if (strlen(utl_acct) >= tListLen(dst->acctId)) { return -1; } - tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + tstrncpy(dst->acctId, utl_acct, tListLen(dst->acctId)); assert(strlen(dst->acctId) > 0); diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index ee940531e6672e1c9b30a6db037d62bc8eac922b..89a86514b5a3c00f37125ef7739601fdbf0ea8f4 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -258,8 +258,8 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { float *data = (float *)pData; - float fmin = FLT_MAX; - float fmax = -FLT_MAX; + float utl_fmin = FLT_MAX; + float utl_fmax = -FLT_MAX; double dsum = 0; *minIndex = 0; *maxIndex = 0; @@ -275,20 +275,20 @@ static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int6 float fv = GET_FLOAT_VAL((const char*)&(data[i])); dsum += fv; - if (fmin > fv) { - fmin = fv; + if (utl_fmin > fv) { + utl_fmin = fv; *minIndex = i; } - if (fmax < fv) { - fmax = fv; + if (utl_fmax < fv) { + utl_fmax = fv; *maxIndex = i; } } SET_DOUBLE_VAL(sum, dsum); - SET_DOUBLE_VAL(max, fmax); - SET_DOUBLE_VAL(min, fmin); + SET_DOUBLE_VAL(max, utl_fmax); + SET_DOUBLE_VAL(min, utl_fmin); } static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 42ece19588d357ef551a4ecb9329dea246d2bae4..1bfad16a2aced5ebd77e89a8d6b5e46880b6d12a 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -16,6 +16,7 @@ #include "hash.h" #include "taos.h" +#include "taoserror.h" #include "taosdef.h" #include "ttoken.h" #include "ttokendef.h" @@ -24,6 +25,10 @@ #include "tvariant.h" void tVariantCreate(tVariant *pVar, SStrToken *token) { + tVariantCreateExt(pVar, token, TK_ID); +} + +void tVariantCreateExt(tVariant *pVar, SStrToken *token, int32_t optrType) { int32_t ret = 0; int32_t type = token->type; @@ -47,7 +52,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_INT:{ ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true); - if (ret != 0) { + if (ret != TSDB_CODE_SUCCESS) { SStrToken t = {0}; tGetToken(token->z, &t.type); if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN @@ -57,7 +62,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { // data overflow, try unsigned parse the input number ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, false); - if (ret != 0) { + if (ret != TSDB_CODE_SUCCESS) { pVar->nType = -1; // -1 means error type return; } @@ -78,15 +83,29 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { break; } case TSDB_DATA_TYPE_TIMESTAMP: { - pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); - break; - } - + if (optrType == TK_NOW) { + pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); + } else if (optrType == TK_PLUS || optrType == TK_MINUS) { + char unit = 0; + ret = parseAbsoluteDuration(token->z, token->n, &pVar->i64, &unit, TSDB_TIME_PRECISION_NANO); + if (ret != TSDB_CODE_SUCCESS) { + pVar->nType = -1; // -1 means error type + return; + } + if (optrType == TK_PLUS) { + pVar->i64 += taosGetTimestamp(TSDB_TIME_PRECISION_NANO); + } else { + pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO) - pVar->i64; + } + } + break; + } + default: { // nType == 0 means the null value type = TSDB_DATA_TYPE_NULL; } } - + pVar->nType = type; } @@ -157,7 +176,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; - + break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method @@ -166,12 +185,12 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 pVar->nLen = (int32_t)len; break; } - + default: pVar->i64 = GET_INT32_VAL(pz); pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes; } - + pVar->nType = type; } diff --git a/src/connector/go b/src/connector/go index 050667e5b4d0eafa5387e4283e713559b421203f..b8f76da4a708d158ec3cc4b844571dc4414e36b4 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f +Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 72ecc6d636453c6a9b71d78d3edce385165cc35f..792ef7c3036f15068796e09883d3f4d47a038fe2 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 72ecc6d636453c6a9b71d78d3edce385165cc35f +Subproject commit 792ef7c3036f15068796e09883d3f4d47a038fe2 diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension index b62a26ecc164a310104df57691691b237e091c89..ce5201014136503d34fecbd56494b67b4961056c 160000 --- a/src/connector/hivemq-tdengine-extension +++ b/src/connector/hivemq-tdengine-extension @@ -1 +1 @@ -Subproject commit b62a26ecc164a310104df57691691b237e091c89 +Subproject commit ce5201014136503d34fecbd56494b67b4961056c diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 6b9fc9d96ce16700ee1243ef7c148a423a965d0b..13b59928dfa9c769bac5b7a26b4bb8e8aa904d48 100644 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -112,6 +112,7 @@ **/*Test.java + **/HttpClientPoolUtilTest.java **/AppMemoryLeakTest.java **/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java **/DatetimeBefore1970Test.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java index 2207db6f9379595e68b8ed00ea8f7298ca3b45ad..1c380fed7dac2c54655830eef6f575e9c07e22af 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -31,6 +31,8 @@ public class TSDBErrorNumbers { public static final int ERROR_RESTFul_Client_IOException = 0x2318; public static final int ERROR_USER_IS_REQUIRED = 0x2319; // user is required public static final int ERROR_PASSWORD_IS_REQUIRED = 0x231a; // password is required + public static final int ERROR_INVALID_JSON_FORMAT = 0x231b; + public static final int ERROR_HTTP_ENTITY_IS_NULL = 0x231c; //http entity is null public static final int ERROR_UNKNOWN = 0x2350; //unknown error @@ -72,6 +74,8 @@ public class TSDBErrorNumbers { errorNumbers.add(ERROR_RESTFul_Client_IOException); errorNumbers.add(ERROR_USER_IS_REQUIRED); errorNumbers.add(ERROR_PASSWORD_IS_REQUIRED); + errorNumbers.add(ERROR_INVALID_JSON_FORMAT); + errorNumbers.add(ERROR_HTTP_ENTITY_IS_NULL); errorNumbers.add(ERROR_RESTFul_Client_Protocol_Exception); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 21c76f73b287e55ef14f5d70cf6a911a9cb543db..eba92741dd8bb3c20f114e35c9a8190193139988 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -66,7 +66,11 @@ public class RestfulStatement extends AbstractStatement { boolean result = true; if (SqlSyntaxValidator.isUseSql(sql)) { - HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); + String ret = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); + JSONObject resultJson = JSON.parseObject(ret); + if (resultJson.getString("status").equals("error")) { + throw TSDBError.createSQLException(resultJson.getInteger("code"), "sql: " + sql + ", desc: " + resultJson.getString("desc")); + } this.database = sql.trim().replace("use", "").trim(); this.conn.setCatalog(this.database); result = false; @@ -115,7 +119,7 @@ public class RestfulStatement extends AbstractStatement { String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); JSONObject resultJson = JSON.parseObject(result); if (resultJson.getString("status").equals("error")) { - throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc")); + throw TSDBError.createSQLException(resultJson.getInteger("code"), "sql: " + sql + ", desc: " + resultJson.getString("desc")); } this.resultSet = new RestfulResultSet(database, this, resultJson); this.affectedRows = 0; @@ -126,7 +130,7 @@ public class RestfulStatement extends AbstractStatement { String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); JSONObject jsonObject = JSON.parseObject(result); if (jsonObject.getString("status").equals("error")) { - throw TSDBError.createSQLException(jsonObject.getInteger("code"), jsonObject.getString("desc")); + throw TSDBError.createSQLException(jsonObject.getInteger("code"), "sql: " + sql + ", desc: " + jsonObject.getString("desc")); } this.resultSet = null; this.affectedRows = getAffectedRows(jsonObject); @@ -134,16 +138,13 @@ public class RestfulStatement extends AbstractStatement { } private int getAffectedRows(JSONObject jsonObject) throws SQLException { - // create ... SQLs should return 0 , and Restful result like this: - // {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1} JSONArray head = jsonObject.getJSONArray("head"); if (head.size() != 1 || !"affected_rows".equals(head.getString(0))) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variable: [" + head.toJSONString() + "]"); JSONArray data = jsonObject.getJSONArray("data"); if (data != null) return data.getJSONArray(0).getInteger(0); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variable: [" + jsonObject.toJSONString() + "]"); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java index de26ab7f1f458a4587ce15bebab3c2c1b0dbc070..99e46bc64f44f6326aec12734849cc5ef518c903 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java @@ -5,6 +5,7 @@ import com.taosdata.jdbc.TSDBErrorNumbers; import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; import org.apache.http.HttpEntity; +import org.apache.http.NoHttpResponseException; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.config.RequestConfig; @@ -30,12 +31,12 @@ import java.sql.SQLException; public class HttpClientPoolUtil { private static final String DEFAULT_CONTENT_TYPE = "application/json"; - private static final int DEFAULT_MAX_TOTAL = 200; - private static final int DEFAULT_MAX_PER_ROUTE = 20; - private static final int DEFAULT_TIME_OUT = 15000; - private static final int DEFAULT_HTTP_KEEP_TIME = 15000; private static final int DEFAULT_MAX_RETRY_COUNT = 5; + private static final int DEFAULT_MAX_TOTAL = 50; + private static final int DEFAULT_MAX_PER_ROUTE = 5; + private static final int DEFAULT_HTTP_KEEP_TIME = -1; + private static final ConnectionKeepAliveStrategy DEFAULT_KEEP_ALIVE_STRATEGY = (response, context) -> { HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); while (it.hasNext()) { @@ -52,29 +53,19 @@ public class HttpClientPoolUtil { return DEFAULT_HTTP_KEEP_TIME * 1000; }; - private static final HttpRequestRetryHandler retryHandler = (exception, executionCount, httpContext) -> { - if (executionCount >= DEFAULT_MAX_RETRY_COUNT) - // do not retry if over max retry count - return false; - if (exception instanceof InterruptedIOException) - // timeout - return false; - if (exception instanceof UnknownHostException) - // unknown host - return false; - if (exception instanceof SSLException) - // SSL handshake exception - return false; - return true; - }; - private static CloseableHttpClient httpClient; static { + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL); connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE); - httpClient = HttpClients.custom().setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY).setConnectionManager(connectionManager).setRetryHandler(retryHandler).build(); + + httpClient = HttpClients.custom() + .setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY) + .setConnectionManager(connectionManager) + .setRetryHandler((exception, executionCount, httpContext) -> executionCount < DEFAULT_MAX_RETRY_COUNT) + .build(); } /*** execute GET request ***/ @@ -118,9 +109,10 @@ public class HttpClientPoolUtil { HttpContext context = HttpClientContext.create(); CloseableHttpResponse httpResponse = httpClient.execute(method, context); httpEntity = httpResponse.getEntity(); - if (httpEntity != null) { - responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8); + if (httpEntity == null) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_HTTP_ENTITY_IS_NULL, "httpEntity is null, sql: " + data); } + responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8); } catch (ClientProtocolException e) { e.printStackTrace(); throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_Protocol_Exception, e.getMessage()); @@ -139,9 +131,6 @@ public class HttpClientPoolUtil { private static HttpRequestBase getRequest(String uri, String methodName) { HttpRequestBase method; RequestConfig requestConfig = RequestConfig.custom() - .setSocketTimeout(DEFAULT_TIME_OUT * 1000) - .setConnectTimeout(DEFAULT_TIME_OUT * 1000) - .setConnectionRequestTimeout(DEFAULT_TIME_OUT * 1000) .setExpectContinueEnabled(false) .build(); if (HttpPut.METHOD_NAME.equalsIgnoreCase(methodName)) { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index 9cf61903f001e84f237e25c3c10fdbb8aac28fd7..e1c4bddb2812f658336c895249886f603681e632 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -121,7 +121,7 @@ public class Utils { } private static void findValuesClauseRangeSet(String preparedSql, RangeSet clauseRangeSet) { - Matcher matcher = Pattern.compile("(values|,)\\s*(\\([^)]*\\))").matcher(preparedSql); + Matcher matcher = Pattern.compile("(values||,)\\s*(\\([^)]*\\))").matcher(preparedSql); while (matcher.find()) { int start = matcher.start(2); int end = matcher.end(2); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java index 0ea46dade29316b99447a6ea4e372bc8057670e8..5ff4b1a41232f57eb21d8394d62cc2dcd27c9cf3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java @@ -57,53 +57,33 @@ public class AuthenticationTest { @Ignore @Test - public void test() { + public void test() throws SQLException { // change password - try { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=taosdata"); - Statement stmt = conn.createStatement(); - stmt.execute("alter user " + user + " pass '" + password + "'"); - stmt.close(); - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=" + user + "&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass '" + password + "'"); + stmt.close(); + conn.close(); + // use new to login and execute query - try { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); - Statement stmt = conn.createStatement(); - stmt.execute("show databases"); - ResultSet rs = stmt.getResultSet(); - ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); - } - System.out.println(); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=" + user + "&password=" + password); + stmt = conn.createStatement(); + stmt.execute("show databases"); + ResultSet rs = stmt.getResultSet(); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); } - } catch (SQLException e) { - e.printStackTrace(); + System.out.println(); } - // change password back - try { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); - Statement stmt = conn.createStatement(); - stmt.execute("alter user " + user + " pass 'taosdata'"); - stmt.close(); - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - - } - @Before - public void before() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } + // change password back + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=" + user + "&password=" + password); + stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass 'taosdata'"); + stmt.close(); + conn.close(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java index d8d8f1eae42f555b2bc0be12e8a3a943071da3c6..5f821c5cc34dde0050d8e62afb6fc8ab17534a17 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java @@ -1,7 +1,5 @@ package com.taosdata.jdbc.cases; -import com.taosdata.jdbc.TSDBErrorNumbers; -import org.junit.Assert; import org.junit.Test; import java.sql.DriverManager; @@ -9,16 +7,14 @@ import java.sql.SQLException; public class ConnectWrongDatabaseTest { - @Test - public void connect() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - DriverManager.getConnection("jdbc:TAOS://localhost:6030/wrong_db?user=root&password=taosdata"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } catch (SQLException e) { - Assert.assertEquals(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL, e.getErrorCode()); - } + @Test(expected = SQLException.class) + public void connectByJni() throws SQLException { + DriverManager.getConnection("jdbc:TAOS://localhost:6030/wrong_db?user=root&password=taosdata"); + } + + @Test(expected = SQLException.class) + public void connectByRestful() throws SQLException { + DriverManager.getConnection("jdbc:TAOS-RS://localhost:6041/wrong_db?user=root&password=taosdata"); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java index beea990456ec98c2ab51fc2086034e0b31b570b6..05c7b0feca21f3f5b9062f9cbc26921aa607732a 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java @@ -18,9 +18,8 @@ public class InsertDbwithoutUseDbTest { private static final Random random = new Random(System.currentTimeMillis()); @Test - public void case001() throws ClassNotFoundException, SQLException { + public void case001() throws SQLException { // prepare schema - Class.forName("com.taosdata.jdbc.TSDBDriver"); String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); try (Statement stmt = conn.createStatement()) { @@ -51,9 +50,8 @@ public class InsertDbwithoutUseDbTest { } @Test - public void case002() throws ClassNotFoundException, SQLException { + public void case002() throws SQLException { // prepare the schema - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); final String url = "jdbc:TAOS-RS://" + host + ":6041/inWithoutDb?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); try (Statement stmt = conn.createStatement()) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java index 1c5c03aacb5e7ed5683c75414975224a67d49e21..d50b56b434fba5795750ec2d92af9f73ae0a02d3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java @@ -374,22 +374,17 @@ public class RestfulConnectionTest { } @BeforeClass - public static void beforeClass() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/log?user=root&password=taosdata", properties); - // create test database for test cases - try (Statement stmt = conn.createStatement()) { - stmt.execute("create database if not exists test"); - } - - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); + public static void beforeClass() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata", properties); + // create test database for test cases + try (Statement stmt = conn.createStatement()) { + stmt.execute("create database if not exists test"); } + } @AfterClass diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index b07dae8003d6e2fea073c0d240f59fb6db0c593f..4f727d43a01ce3c4c459fcd2d7328e1c4b132976 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -10,11 +10,11 @@ import java.util.Random; public class RestfulJDBCTest { private static final String host = "127.0.0.1"; - private final Random random = new Random(System.currentTimeMillis()); - private Connection connection; + private static final Random random = new Random(System.currentTimeMillis()); + private static Connection connection; @Test - public void testCase001() { + public void testCase001() throws SQLException { // given String sql = "drop database if exists restful_test"; // when @@ -38,7 +38,7 @@ public class RestfulJDBCTest { } @Test - public void testCase002() { + public void testCase002() throws SQLException { // given String sql = "create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; // when @@ -48,7 +48,7 @@ public class RestfulJDBCTest { } @Test - public void testCase004() { + public void testCase004() throws SQLException { for (int i = 1; i <= 100; i++) { // given String sql = "create table t" + i + " using weather tags('beijing', '" + i + "')"; @@ -60,7 +60,7 @@ public class RestfulJDBCTest { } @Test - public void testCase005() { + public void testCase005() throws SQLException { int rows = 0; for (int i = 0; i < 10; i++) { for (int j = 1; j <= 100; j++) { @@ -99,7 +99,7 @@ public class RestfulJDBCTest { } @Test - public void testCase007() { + public void testCase007() throws SQLException { // given String sql = "drop database restful_test"; @@ -110,50 +110,41 @@ public class RestfulJDBCTest { Assert.assertFalse(execute); } - private int executeUpdate(Connection connection, String sql) { + private int executeUpdate(Connection connection, String sql) throws SQLException { try (Statement stmt = connection.createStatement()) { return stmt.executeUpdate(sql); - } catch (SQLException e) { - e.printStackTrace(); } - return 0; } - private boolean execute(Connection connection, String sql) { + private boolean execute(Connection connection, String sql) throws SQLException { try (Statement stmt = connection.createStatement()) { return stmt.execute(sql); - } catch (SQLException e) { - e.printStackTrace(); } - return false; } - private ResultSet executeQuery(Connection connection, String sql) { + private ResultSet executeQuery(Connection connection, String sql) throws SQLException { try (Statement statement = connection.createStatement()) { return statement.executeQuery(sql); - } catch (SQLException e) { - e.printStackTrace(); } - return null; } - @Before - public void before() { + @BeforeClass + public static void beforeClass() { try { - connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); } catch (SQLException e) { e.printStackTrace(); } } - @After - public void after() { - try { - if (connection != null) - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); + @AfterClass + public static void afterClass() throws SQLException { + if (connection != null) { + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists restful_test"); + stmt.close(); + connection.close(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java index c7fc81297264f3cf38795d9d5a3b7eccc51574c9..f3011af799c987ed399920875ae512fd8533ec77 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -186,22 +186,17 @@ public class RestfulResultSetMetaDataTest { } @BeforeClass - public static void beforeClass() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); - stmt = conn.createStatement(); - stmt.execute("create database if not exists restful_test"); - stmt.execute("use restful_test"); - stmt.execute("drop table if exists weather"); - stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); - stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); - rs = stmt.executeQuery("select * from restful_test.weather"); - rs.next(); - meta = rs.getMetaData(); - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); - } + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + meta = rs.getMetaData(); } @AfterClass diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index 21a91669b270df4dc2e8f7b4885fb9e8eedbfdf7..8ffa1de21c604c2235c23e63e93b008f5a1d146c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -658,36 +658,29 @@ public class RestfulResultSetTest { } @BeforeClass - public static void beforeClass() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); - stmt = conn.createStatement(); - stmt.execute("create database if not exists restful_test"); - stmt.execute("use restful_test"); - stmt.execute("drop table if exists weather"); - stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); - stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); - rs = stmt.executeQuery("select * from restful_test.weather"); - rs.next(); - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); - } - + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("drop database if exists restful_test"); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); } @AfterClass - public static void afterClass() { - try { - if (rs != null) - rs.close(); - if (stmt != null) - stmt.close(); - if (conn != null) - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); + public static void afterClass() throws SQLException { + if (rs != null) + rs.close(); + if (stmt != null) { + stmt.execute("drop database if exists restful_test"); + stmt.close(); } + if (conn != null) + conn.close(); } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index 98482ade80656f2e48bc6927953439cfe4b010c1..7dff90a77556f029049a4216e789dc64e42a2e96 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -581,11 +581,14 @@ public class SQLTest { @BeforeClass public static void before() throws SQLException { - connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); } @AfterClass public static void after() throws SQLException { + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists restful_test"); + stmt.close(); connection.close(); } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/HttpClientPoolUtilTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/HttpClientPoolUtilTest.java new file mode 100644 index 0000000000000000000000000000000000000000..cae33f18e7a04e443092d8e696bb32be9600a435 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/HttpClientPoolUtilTest.java @@ -0,0 +1,76 @@ +package com.taosdata.jdbc.utils; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBError; +import org.junit.Test; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class HttpClientPoolUtilTest { + + String user = "root"; + String password = "taosdata"; + String host = "127.0.0.1"; + String dbname = "log"; + + @Test + public void test() { + // given + List threads = IntStream.range(0, 4000).mapToObj(i -> new Thread(() -> { + useDB(); +// try { +// TimeUnit.SECONDS.sleep(10); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } + })).collect(Collectors.toList()); + + threads.forEach(Thread::start); + + for (Thread thread : threads) { + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + } + + private void useDB() { + try { + user = URLEncoder.encode(user, StandardCharsets.UTF_8.displayName()); + password = URLEncoder.encode(password, StandardCharsets.UTF_8.displayName()); + String loginUrl = "http://" + host + ":" + 6041 + "/rest/login/" + user + "/" + password + ""; + String result = HttpClientPoolUtil.execute(loginUrl); + JSONObject jsonResult = JSON.parseObject(result); + String status = jsonResult.getString("status"); + String token = jsonResult.getString("desc"); + if (!status.equals("succ")) { + throw new SQLException(jsonResult.getString("desc")); + } + + String url = "http://" + host + ":6041/rest/sql"; + String sql = "use " + dbname; + result = HttpClientPoolUtil.execute(url, sql, token); + + JSONObject resultJson = JSON.parseObject(result); + if (resultJson.getString("status").equals("error")) { + throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc")); + } + } catch (UnsupportedEncodingException | SQLException e) { + e.printStackTrace(); + } + } + + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java index 1cbd95b2492284b9c85f31bd6b6848d9c223df18..66ecb9d63beb7c57ffb992a9ba5999b8fb70e739 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java @@ -73,6 +73,48 @@ public class UtilsTest { Assert.assertEquals(expected, actual); } + @Test + public void multiValuesAndWhitespace() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?) (?,?,?,?) (?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4) (200,3.1415,'xyz',5) (300,3.141592,'uvw',6)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void multiValuesNoSeparator() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?)(?,?,?,?)(?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4)(200,3.1415,'xyz',5)(300,3.141592,'uvw',6)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void multiValuesMultiSeparator() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?) (?,?,?,?), (?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4) (200,3.1415,'xyz',5), (300,3.141592,'uvw',6)"; + Assert.assertEquals(expected, actual); + } + @Test public void lineTerminator() { // given @@ -100,6 +142,32 @@ public class UtilsTest { Assert.assertEquals(expected, actual); } + @Test + public void lineTerminatorAndMultiValuesAndNoneOrMoreWhitespace() { + String nativeSql = "INSERT Into ? TAGS(?) VALUES(?,?,\r\n?,?),(?,? ,\r\n?,?) t? tags (?) Values (?,?,?\r\n,?) (?,?,?,?) t? Tags(?) values (?,?,?,?) , (?,?,?,?)"; + Object[] parameters = Stream.of("t1", "abc", 100, 1.1, "xxx", "xxx", 200, 2.2, "xxx", "xxx", 2, "bcd", 300, 3.3, "xxx", "xxx", 400, 4.4, "xxx", "xxx", 3, "cde", 500, 5.5, "xxx", "xxx", 600, 6.6, "xxx", "xxx").toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT Into t1 TAGS('abc') VALUES(100,1.1,\r\n'xxx','xxx'),(200,2.2 ,\r\n'xxx','xxx') t2 tags ('bcd') Values (300,3.3,'xxx'\r\n,'xxx') (400,4.4,'xxx','xxx') t3 Tags('cde') values (500,5.5,'xxx','xxx') , (600,6.6,'xxx','xxx')"; + Assert.assertEquals(expected, actual); + } + + @Test + public void multiValuesAndNoneOrMoreWhitespace() { + String nativeSql = "INSERT INTO ? USING traces TAGS (?, ?) VALUES (?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?)"; + Object[] parameters = Stream.of("t1", "t1", "t2", 1632968284000L, 111.111, 119.001, 0.4, 90, 99.1, "WGS84", 1632968285000L, 111.21109999999999, 120.001, 0.5, 91, 99.19999999999999, "WGS84").toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO t1 USING traces TAGS ('t1', 't2') VALUES (1632968284000, 111.111, 119.001, 0.4, 90, 99.1, 'WGS84') (1632968285000, 111.21109999999999, 120.001, 0.5, 91, 99.19999999999999, 'WGS84')"; + Assert.assertEquals(expected, actual); + } + @Test public void replaceNothing() { // given diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index a1b6fe312b5725b8bf030701608d93c3e0c85706..a9bdba33e8212b7a32ef2c6b518aef03aed3f50a 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -103,7 +103,7 @@ _libtaos.taos_get_client_info.restype = c_char_p def taos_get_client_info(): # type: () -> str """Get client version info.""" - return _libtaos.taos_get_client_info().decode() + return _libtaos.taos_get_client_info().decode("utf-8") _libtaos.taos_get_server_info.restype = c_char_p @@ -113,7 +113,7 @@ _libtaos.taos_get_server_info.argtypes = (c_void_p,) def taos_get_server_info(connection): # type: (c_void_p) -> str """Get server version as string.""" - return _libtaos.taos_get_server_info(connection).decode() + return _libtaos.taos_get_server_info(connection).decode("utf-8") _libtaos.taos_close.restype = None diff --git a/src/connector/python/taos/field.py b/src/connector/python/taos/field.py index b0bec58b932f2136b868739bb28fca04de759e3f..f6fa28e8336fa53a137ce7bd9a25b99a32263f1b 100644 --- a/src/connector/python/taos/field.py +++ b/src/connector/python/taos/field.py @@ -144,7 +144,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_ try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append(tmpstr.value.decode()) + res.append(tmpstr.value.decode("utf-8")) else: res.append( ( @@ -172,7 +172,7 @@ def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=Field if rbyte == 1 and buffer[0] == b'\xff': res.append(None) else: - res.append(cast(buffer, c_char_p).value.decode()) + res.append(cast(buffer, c_char_p).value.decode("utf-8")) return res @@ -188,7 +188,7 @@ def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldT if rbyte == 4 and buffer[:4] == b'\xff'*4: res.append(None) else: - res.append(cast(buffer, c_char_p).value.decode()) + res.append(cast(buffer, c_char_p).value.decode("utf-8")) return res diff --git a/src/connector/python/taos/result.py b/src/connector/python/taos/result.py index 81151733615d1b7fdc3318b6e53888ae39d32b14..8881c26ca94edfd4ed3a5042bebbc327f06b82ac 100644 --- a/src/connector/python/taos/result.py +++ b/src/connector/python/taos/result.py @@ -3,6 +3,8 @@ from .cinterface import * # from .connection import TaosConnection from .error import * +from ctypes import c_void_p + class TaosResult(object): """TDengine result interface""" @@ -12,7 +14,11 @@ class TaosResult(object): # to make the __del__ order right self._conn = conn self._close_after = close_after - self._result = result + if isinstance(result, c_void_p): + self._result = result + else: + self._result = c_void_p(result) + self._fields = None self._field_count = None self._precision = None diff --git a/src/connector/python/tests/test_lines.py b/src/connector/python/tests/test_lines.py index bd9d2cdb39d6f4f2612581ce7284c057c456ef91..c9b581e8bcb0d300ade8cb991ba648efc2550f8a 100644 --- a/src/connector/python/tests/test_lines.py +++ b/src/connector/python/tests/test_lines.py @@ -36,7 +36,6 @@ def test_insert_lines(conn): conn.insert_lines(lines) print("inserted") result = conn.query("select * from st") - print(*result.fields) all = result.rows_iter() for row in all: print(row) diff --git a/src/connector/python/tests/test_stmt.py b/src/connector/python/tests/test_stmt.py index 938ba10eb3d2377a63f7972deb99dbd47f7de1b2..3368ecb6a9336a4295790f2cd55314ac9bb6290e 100644 --- a/src/connector/python/tests/test_stmt.py +++ b/src/connector/python/tests/test_stmt.py @@ -1,3 +1,4 @@ +# encoding:UTF-8 from taos import * from ctypes import * diff --git a/src/connector/python/tests/test_stream.py b/src/connector/python/tests/test_stream.py index de6e20928b176e51bc6d350fb01268459f4e7f95..32ec4c5999c975be907cf69a42a04b5f4dd5d54c 100644 --- a/src/connector/python/tests/test_stream.py +++ b/src/connector/python/tests/test_stream.py @@ -20,7 +20,8 @@ def stream_callback(p_param, p_result, p_row): result = TaosResult(p_result) row = TaosRow(result, p_row) try: - ts, count = row() + ts, count = row.as_tuple() + print(ts, count) p = cast(p_param, POINTER(Counter)) p.contents.count += count print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index aac5a1c665c1417069b3978d10c7b1406a6b02a4..65a6b5a7c8bebbadca31730983910a615f620766 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -424,7 +424,7 @@ static void cqProcessCreateTimer(void *param, void *tmrId) { // inner implement in tscStream.c TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* desName, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), - int64_t stime, void *param, void (*callback)(void *), void* cqhandle); + int64_t tsc_stime, void *param, void (*callback)(void *), void* cqhandle); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { pObj->pContext = pContext; diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 87baff30673afc68eb23a00bef279433a422ba67..5612a1f74206045bd3e6c9c4dbd4c7cfff26c668 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -229,12 +229,12 @@ static void dnodeAllocCheckItem() { } void dnodeCleanupCheck() { - for (ECheckItemType index = 0; index < TSDB_CHECK_ITEM_MAX; ++index) { - if (tsCheckItem[index].enable && tsCheckItem[index].stopFp) { - (*tsCheckItem[index].stopFp)(); + for (ECheckItemType dnd_index = 0; dnd_index < TSDB_CHECK_ITEM_MAX; ++dnd_index) { + if (tsCheckItem[dnd_index].enable && tsCheckItem[dnd_index].stopFp) { + (*tsCheckItem[dnd_index].stopFp)(); } - if (tsCheckItem[index].cleanUpFp) { - (*tsCheckItem[index].cleanUpFp)(); + if (tsCheckItem[dnd_index].cleanUpFp) { + (*tsCheckItem[dnd_index].cleanUpFp)(); } } } @@ -242,19 +242,19 @@ void dnodeCleanupCheck() { int32_t dnodeInitCheck() { dnodeAllocCheckItem(); - for (ECheckItemType index = 0; index < TSDB_CHECK_ITEM_MAX; ++index) { - if (tsCheckItem[index].initFp) { - if ((*tsCheckItem[index].initFp)() != 0) { - dError("failed to init check item:%s", tsCheckItem[index].name); + for (ECheckItemType dnd_index = 0; dnd_index < TSDB_CHECK_ITEM_MAX; ++dnd_index) { + if (tsCheckItem[dnd_index].initFp) { + if ((*tsCheckItem[dnd_index].initFp)() != 0) { + dError("failed to init check item:%s", tsCheckItem[dnd_index].name); return -1; } } } - for (ECheckItemType index = 0; index < TSDB_CHECK_ITEM_MAX; ++index) { - if (tsCheckItem[index].enable && tsCheckItem[index].startFp) { - if ((*tsCheckItem[index].startFp)() != 0) { - dError("failed to check item:%s", tsCheckItem[index].name); + for (ECheckItemType dnd_index = 0; dnd_index < TSDB_CHECK_ITEM_MAX; ++dnd_index) { + if (tsCheckItem[dnd_index].enable && tsCheckItem[dnd_index].startFp) { + if ((*tsCheckItem[dnd_index].startFp)() != 0) { + dError("failed to check item:%s", tsCheckItem[dnd_index].name); exit(-1); } } diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 08269c0bf6141974366936660bee326682cd90f5..c2bc619fc6b5b347d1e39cfe713451f7e726ddc9 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -56,17 +56,17 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue; - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = tsDnodeDnodePort; - rpcInit.label = "DND-S"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dnodeProcessReqMsgFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - - tsServerRpc = rpcOpen(&rpcInit); + SRpcInit dnd_rpcInit; + memset(&dnd_rpcInit, 0, sizeof(dnd_rpcInit)); + dnd_rpcInit.localPort = tsDnodeDnodePort; + dnd_rpcInit.label = "DND-S"; + dnd_rpcInit.numOfThreads = 1; + dnd_rpcInit.cfp = dnodeProcessReqMsgFromDnode; + dnd_rpcInit.sessions = TSDB_MAX_VNODES << 4; + dnd_rpcInit.connType = TAOS_CONN_SERVER; + dnd_rpcInit.idleTime = tsShellActivityTimer * 1000; + + tsServerRpc = rpcOpen(&dnd_rpcInit); if (tsServerRpc == NULL) { dError("failed to init inter-dnodes RPC server"); return -1; @@ -123,19 +123,19 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { int32_t dnodeInitClient() { char secret[TSDB_KEY_LEN] = "secret"; - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dnodeProcessRspFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = "t"; - rpcInit.ckey = "key"; - rpcInit.secret = secret; - - tsClientRpc = rpcOpen(&rpcInit); + SRpcInit dnd_rpcInit; + memset(&dnd_rpcInit, 0, sizeof(dnd_rpcInit)); + dnd_rpcInit.label = "DND-C"; + dnd_rpcInit.numOfThreads = 1; + dnd_rpcInit.cfp = dnodeProcessRspFromDnode; + dnd_rpcInit.sessions = TSDB_MAX_VNODES << 4; + dnd_rpcInit.connType = TAOS_CONN_CLIENT; + dnd_rpcInit.idleTime = tsShellActivityTimer * 1000; + dnd_rpcInit.user = "t"; + dnd_rpcInit.ckey = "key"; + dnd_rpcInit.secret = secret; + + tsClientRpc = rpcOpen(&dnd_rpcInit); if (tsClientRpc == NULL) { dError("failed to init mnode rpc client"); return -1; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 5606681f0f931070e9cbf21d6b98b0d2eb51bdfa..37f53f731a92f352049e081c8ecedee32f5a3400 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -83,18 +83,18 @@ int32_t dnodeInitShell() { numOfThreads = 1; } - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = tsDnodeShellPort; - rpcInit.label = "SHELL"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dnodeProcessMsgFromShell; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dnodeRetrieveUserAuthInfo; - - tsShellRpc = rpcOpen(&rpcInit); + SRpcInit dnd_rpcInit; + memset(&dnd_rpcInit, 0, sizeof(dnd_rpcInit)); + dnd_rpcInit.localPort = tsDnodeShellPort; + dnd_rpcInit.label = "SHELL"; + dnd_rpcInit.numOfThreads = numOfThreads; + dnd_rpcInit.cfp = dnodeProcessMsgFromShell; + dnd_rpcInit.sessions = tsMaxShellConns; + dnd_rpcInit.connType = TAOS_CONN_SERVER; + dnd_rpcInit.idleTime = tsShellActivityTimer * 1000; + dnd_rpcInit.afp = dnodeRetrieveUserAuthInfo; + + tsShellRpc = rpcOpen(&dnd_rpcInit); if (tsShellRpc == NULL) { dError("failed to init shell rpc server"); return -1; diff --git a/src/inc/query.h b/src/inc/query.h index fb9cbff8584892b4a6bc6e4a6ce046a7500aef39..0872e3dbaa517ded77dd758b30e69f273c13a580 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -76,6 +76,11 @@ void* qGetResultRetrieveMsg(qinfo_t qinfo); */ int32_t qKillQuery(qinfo_t qinfo); +//kill by qid +int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount); + +bool qSolveCommitNoBlock(void* pRepo, void* pMgmt); + int32_t qQueryCompleted(qinfo_t qinfo); /** diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 8fd6fd29ed7875e7b17ecf627755259ebe3fe163..5a030cb6006ab3d31566dc4ed3ede8996b514e7f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -79,12 +79,17 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DATA_NULL_STR_L "null" #define TSDB_DEFAULT_USER "root" + #ifdef _TD_POWER_ #define TSDB_DEFAULT_PASS "powerdb" #elif (_TD_TQ_ == true) #define TSDB_DEFAULT_PASS "tqueue" #elif (_TD_PRO_ == true) #define TSDB_DEFAULT_PASS "prodb" +#elif (_TD_KH_ == true) +#define TSDB_DEFAULT_PASS "khroot" +#elif (_TD_JH_ == true) +#define TSDB_DEFAULT_PASS "jhdata" #else #define TSDB_DEFAULT_PASS "taosdata" #endif diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 000703464cfb8c687e473b7559e1048c42f8a6de..43ed20fe14af7c5f60c580dc08ff8456e7903e65 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -35,6 +35,7 @@ int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) #define TSDB_CODE_SUCCESS 0 +#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //"Action in progress") @@ -106,6 +107,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") #define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") +#define TSDB_CODE_TSC_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0227) //"Result set too large to be output") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index dd4374761a6c90aa97b822ba8e814781b8ef0c7c..3d375829178866cdf2dcfb9c79dddfbdae71f86e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -400,7 +400,7 @@ typedef struct SColIndex { int16_t colId; // column id int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag uint16_t flag; // denote if it is a tag or a normal column - char name[TSDB_COL_NAME_LEN + TSDB_DB_NAME_LEN + 1]; + char name[TSDB_COL_NAME_LEN + TSDB_TABLE_NAME_LEN + 1]; } SColIndex; typedef struct SColumnFilterInfo { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 6b68b467a38f55a9e7089f797fc81fbf69ddb93c..8fc050d9e3a5ddcb24183b2d47f406a443512671 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -39,6 +39,7 @@ extern "C" { #define TSDB_STATUS_COMMIT_START 1 #define TSDB_STATUS_COMMIT_OVER 2 +#define TSDB_STATUS_COMMIT_NOBLOCK 3 //commit no block, need to be solved // TSDB STATE DEFINITION #define TSDB_STATE_OK 0x0 @@ -417,6 +418,11 @@ int tsdbSyncRecv(void *pRepo, SOCKET socketFd); // For TSDB Compact int tsdbCompact(STsdbRepo *pRepo); +// For TSDB Health Monitor + +// no problem return true +bool tsdbNoProblem(STsdbRepo* pRepo); + #ifdef __cplusplus } #endif diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 0a84d7f13b735e0589b4df5e17122ed2b0fe69b2..4ab851bc07c7494e681947c38b88513c4d2e38cd 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -135,12 +135,12 @@ #define TK_USING 117 #define TK_NULL 118 #define TK_NOW 119 -#define TK_SELECT 120 -#define TK_UNION 121 -#define TK_ALL 122 -#define TK_DISTINCT 123 -#define TK_FROM 124 -#define TK_VARIABLE 125 +#define TK_VARIABLE 120 +#define TK_SELECT 121 +#define TK_UNION 122 +#define TK_ALL 123 +#define TK_DISTINCT 124 +#define TK_FROM 125 #define TK_INTERVAL 126 #define TK_EVERY 127 #define TK_SESSION 128 @@ -211,6 +211,7 @@ #define TK_INSERT 193 #define TK_INTO 194 #define TK_VALUES 195 +#define TK_FILE 196 #define TK_SPACE 300 @@ -219,7 +220,6 @@ #define TK_HEX 303 // hex number 0x123 #define TK_OCT 304 // oct number #define TK_BIN 305 // bin format data 0b111 -#define TK_FILE 306 #define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query #endif diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 747b1bf28ddf24869dba4db261921f869bedd5e1..3d112415e5d80327b1529604ac0443b6628af2ef 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -26,6 +26,8 @@ ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS) + ADD_DEFINITIONS(-DUNICODE) + ADD_DEFINITIONS(-D_UNICODE) LIST(APPEND SRC ./src/shellEngine.c) LIST(APPEND SRC ./src/shellMain.c) LIST(APPEND SRC ./src/shellWindows.c) @@ -34,8 +36,14 @@ ELSEIF (TD_WINDOWS) IF (TD_POWER) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) + ELSEIF (TD_TQ) + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME tq) ELSEIF (TD_PRO) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME prodbc) + ELSEIF (TD_KH) + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME khclient) + ELSEIF (TD_JH) + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME jh_taos) ELSE () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 03ccfe2d576df76407bc7a22cf17d884dd2bad51..9c5794278c5bd9545fb6260e4f8442d8c9e8cad9 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -28,8 +28,16 @@ #define MAX_HISTORY_SIZE 1000 #define MAX_COMMAND_SIZE 1048586 -#ifdef _TD_PRO_ +#ifdef _TD_POWER_ + #define HISTORY_FILE ".power_history" +#elif (_TD_TQ_ == true) + #define HISTORY_FILE ".tq_history" +#elif (_TD_PRO_ == true) #define HISTORY_FILE ".prodb_history" +#elif (_TD_KH_ == true) + #define HISTORY_FILE ".kh_history" +#elif (_TD_JH_ == true) + #define HISTORY_FILE ".jh_taos_history" #else #define HISTORY_FILE ".taos_history" #endif diff --git a/src/kit/shell/src/shellCheck.c b/src/kit/shell/src/shellCheck.c index 5821281a036674e7a60edc2f63500822a358b1bc..43256719e125a712e6a52ddadaa9637498278092 100644 --- a/src/kit/shell/src/shellCheck.c +++ b/src/kit/shell/src/shellCheck.c @@ -36,6 +36,7 @@ typedef struct { int totalThreads; void * taos; char * db; + int code; } ShellThreadObj; static int32_t shellUseDb(TAOS *con, char *db) { @@ -112,10 +113,10 @@ static void *shellCheckThreadFp(void *arg) { int32_t end = (pThread->threadIndex + 1) * interval; if (start >= tbNum) return NULL; - if (end > tbNum) end = tbNum + 1; + if (end > tbNum) end = tbNum; char file[32] = {0}; - snprintf(file, 32, "tb%d.txt", pThread->threadIndex); + snprintf(file, 32, "tb%d.sql", pThread->threadIndex); FILE *fp = fopen(file, "w"); if (!fp) { @@ -123,16 +124,19 @@ static void *shellCheckThreadFp(void *arg) { return NULL; } + ASSERT(pThread->code != 0); + char sql[SHELL_SQL_LEN]; for (int32_t t = start; t < end; ++t) { char *tbname = tbNames[t]; if (tbname == NULL) break; - snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname); + snprintf(sql, SHELL_SQL_LEN, "select last_row(_c0) from %s;", tbname); TAOS_RES *pSql = taos_query(pThread->taos, sql); int32_t code = taos_errno(pSql); - if (code != 0) { + // -k: -1 means check all errors, while other non-zero values means check specific errors. + if ((code == pThread->code) || ((pThread->code == -1) && (code != 0))) { int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); fwrite(sql, 1, len, fp); atomic_add_fetch_32(&errorNum, 1); @@ -161,6 +165,7 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { pThread->totalThreads = _args->threadNum; pThread->taos = con; pThread->db = _args->database; + pThread->code = _args->check; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index a1413be1ce4ce6f67516fc09121115f30bbc56f0..e9cae7003251f4f10d3ec7cab12a83176929ba09 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -34,10 +34,10 @@ void insertChar(Command *cmd, char *c, int size); void printHelp() { char indent[10] = " "; - printf("taos shell is used to test the TDengine database\n"); + printf("taos shell is used to test the database\n"); printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost."); + printf("%s%s%s\n", indent, indent, "Server IP address to connect. The default host is localhost."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); printf("%s%s\n", indent, "-P"); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index ffd966cb236f37e815efaf70226d57db24a15cf1..26dc24bd1a0173c585fe8b951eb0b3f5154c8814 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -34,28 +34,37 @@ char CLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n" "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n"; char PROMPT_HEADER[] = "power> "; - char CONTINUE_PROMPT[] = " -> "; int prompt_size = 7; #elif (_TD_TQ_ == true) char CLIENT_VERSION[] = "Welcome to the TQ shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TQ, Inc. All rights reserved.\n\n"; char PROMPT_HEADER[] = "tq> "; - -char CONTINUE_PROMPT[] = " -> "; +char CONTINUE_PROMPT[] = " -> "; int prompt_size = 4; #elif (_TD_PRO_ == true) char CLIENT_VERSION[] = "Welcome to the ProDB shell from %s, Client Version:%s\n" "Copyright (c) 2020 by Hanatech, Inc. All rights reserved.\n\n"; char PROMPT_HEADER[] = "ProDB> "; - char CONTINUE_PROMPT[] = " -> "; int prompt_size = 7; +#elif (_TD_KH_ == true) +char CLIENT_VERSION[] = "Welcome to the KingHistorian shell from %s, Client Version:%s\n" + "Copyright (c) 2021 by Wellintech, Inc. All rights reserved.\n\n"; +char PROMPT_HEADER[] = "khclient> "; + +char CONTINUE_PROMPT[] = " -> "; +int prompt_size = 10; +#elif (_TD_JH_ == true) +char CLIENT_VERSION[] = "Welcome to the jh_iot shell from %s, Client Version:%s\n" + "Copyright (c) 2021 by jinheng, Inc. All rights reserved.\n\n"; +char PROMPT_HEADER[] = "jh_taos> "; +char CONTINUE_PROMPT[] = " -> "; +int prompt_size = 9; #else char CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; char PROMPT_HEADER[] = "taos> "; - char CONTINUE_PROMPT[] = " -> "; int prompt_size = 6; #endif @@ -111,7 +120,11 @@ TAOS *shellInit(SShellArguments *_args) { } if (con == NULL) { +#ifdef _TD_KH_ + printf("kh connect failed, reason: %s.\n\n", tstrerror(terrno)); +#else printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno)); +#endif fflush(stdout); return con; } @@ -262,7 +275,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { } if (c == '\\') { - if (quote != 0 && (*command == '_' || *command == '\\')) { + if (quote != 0 && (*command == '_' || *command == '%' || *command == '\\')) { //DO nothing } else { esc = true; diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index 38abb423cfd2c0329dad24244a798f0617b4cbb6..9d3a99bb4bb731ad9e89ed01b2839eebea7d4c7b 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -25,7 +25,7 @@ static char **shellSQLFiles = NULL; static int32_t shellSQLFileNum = 0; -static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0}; +static char shellTablesSQLFile[4096] = {0}; typedef struct { pthread_t threadID; @@ -93,8 +93,8 @@ static void shellCheckTablesSQLFile(const char *directoryName) { sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName); - struct stat fstat; - if (stat(shellTablesSQLFile, &fstat) < 0) { + struct stat kit_fstat; + if (stat(shellTablesSQLFile, &kit_fstat) < 0) { shellTablesSQLFile[0] = 0; } } diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 93783b205560604c9d25c9f5dc2e73a239a67b8e..8d36e1cc6b785c17d90c409a412df2b288d0a620 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -29,12 +29,16 @@ struct termios oldtio; extern int wcwidth(wchar_t c); void insertChar(Command *cmd, char *c, int size); const char *argp_program_version = version; +#ifdef _TD_KH_ +const char *argp_program_bug_address = ""; +#else const char *argp_program_bug_address = ""; +#endif static char doc[] = ""; static char args_doc[] = ""; static struct argp_option options[] = { - {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, - {"password", 'p', 0, 0, "The password to use when connecting to the server."}, + {"host", 'h', "HOST", 0, "Server FQDN to connect. The default host is localhost."}, + {"password", 'p', 0, 0, "The password to use when connecting to the server."}, {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 5c9dc0995dacecebd10b7f2b77e216ca97157db0..afed5d2d2ffa680852c1155334499975cd58cfea 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -95,6 +95,9 @@ SShellArguments args = { */ int main(int argc, char* argv[]) { /*setlocale(LC_ALL, "en_US.UTF-8"); */ +#ifdef WINDOWS + SetConsoleOutputCP(CP_UTF8); +#endif if (!checkVersion()) { exit(EXIT_FAILURE); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index abec34b84c5ff65d6cb13492028cd36321d2d0ca..f693ae0b56cea2da12e01a0c82e32ac739e5ca4d 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -17,10 +17,30 @@ #include "taos.h" #include "shellCommand.h" +#define SHELL_INPUT_MAX_COMMAND_SIZE 10000 + extern char configDir[]; +#ifdef _TD_POWER_ +char WINCLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n"; +#elif (_TD_TQ_ == true) +char WINCLIENT_VERSION[] = "Welcome to the TQ shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; +#elif (_TD_PRO_ == true) +char WINCLIENT_VERSION[] = "Welcome to the ProDB shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by Hanatech, Inc. All rights reserved.\n\n"; +#elif (_TD_KH_ == true) +char WINCLIENT_VERSION[] = "Welcome to the KingHistorian shell from %s, Client Version:%s\n" + "Copyright (c) 2021 by Wellintech, Inc. All rights reserved.\n\n"; +#elif (_TD_JH_ == true) +char WINCLIENT_VERSION[] = "Welcome to the jh_iot shell from %s, Client Version:%s\n" + "Copyright (c) 2021 by jinheng, Inc. All rights reserved.\n\n"; +#else char WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; +#endif + void printVersion() { printf("version: %s\n", version); @@ -28,10 +48,10 @@ void printVersion() { void printHelp() { char indent[10] = " "; - printf("taos shell is used to test the TDengine database\n"); + printf("taos shell is used to test the database\n"); printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost."); + printf("%s%s%s\n", indent, indent, "Server FQDN to connect. The default host is localhost."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); printf("%s%s\n", indent, "-P"); @@ -256,27 +276,37 @@ int32_t shellReadCommand(TAOS *con, char command[]) { cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); // Read input. - char c; + void *console = GetStdHandle(STD_INPUT_HANDLE); + unsigned long read; + wchar_t *c= (wchar_t *)calloc(SHELL_INPUT_MAX_COMMAND_SIZE, sizeof(wchar_t)); + char mbStr[16]; while (1) { - c = getchar(); - - switch (c) { - case '\n': - case '\r': - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - free(cmd.buffer); - cmd.buffer = NULL; - free(cmd.command); - cmd.command = NULL; - return 0; - } else { - shellPrintContinuePrompt(); - updateBuffer(&cmd); - } - break; - default: - insertChar(&cmd, c); + int ret = ReadConsole(console, c, SHELL_INPUT_MAX_COMMAND_SIZE, &read, NULL); + for (int input_index = 0; input_index < read; input_index++) { + int size = WideCharToMultiByte(CP_UTF8, 0, &c[input_index], 1, mbStr, sizeof(mbStr), NULL, NULL); + mbStr[size] = 0; + switch (c[input_index]) { + case '\n': + if (isReadyGo(&cmd)) { + sprintf(command, "%s%s", cmd.buffer, cmd.command); + free(cmd.buffer); + cmd.buffer = NULL; + free(cmd.command); + cmd.command = NULL; + free(c); + return 0; + } else { + shellPrintContinuePrompt(); + updateBuffer(&cmd); + } + break; + case '\r': + break; + default: + for (int i = 0; i < size; ++i) { + insertChar(&cmd, mbStr[i]); + } + } } } @@ -304,6 +334,20 @@ void *shellLoopQuery(void *arg) { return NULL; } -void get_history_path(char *history) { sprintf(history, "C:/TDengine/%s", HISTORY_FILE); } +void get_history_path(char *history) { +#ifdef _TD_POWER_ + sprintf(history, "C:/PowerDB/%s", HISTORY_FILE); +#elif (_TD_TQ_ == true) + sprintf(history, "C:/TQueue/%s", HISTORY_FILE); +#elif (_TD_PRO_ == true) + sprintf(history, "C:/ProDB/%s", HISTORY_FILE); +#elif (_TD_KH_ == true) + sprintf(history, "C:/KingHistorian/%s", HISTORY_FILE); +#elif (_TD_JH_ == true) + sprintf(history, "C:/jh_iot/%s", HISTORY_FILE); +#else + sprintf(history, "C:/TDengine/%s", HISTORY_FILE); +#endif +} void exitShell() { exit(EXIT_SUCCESS); } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8fb0fefa131606f0893312f1f7db5e2d364b10c2..3a0f3a7f61b52f85da1809edd7120b6babe93169 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -75,6 +75,7 @@ extern char configDir[]; #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. +#define MAX_PATH_LEN 4096 #define DEFAULT_START_TIME 1500000000000 @@ -244,6 +245,7 @@ typedef struct SArguments_S { uint64_t insert_interval; uint64_t timestamp_step; int64_t query_times; + int64_t prepared_rand; uint32_t interlaceRows; uint32_t reqPerReq; // num_of_records_per_req uint64_t max_sql_len; @@ -303,6 +305,7 @@ typedef struct SSuperTable_S { uint64_t lenOfTagOfOneRow; char* sampleDataBuf; + bool useSampleTs; uint32_t tagSource; // 0: rand, 1: tag sample char* tagDataBuf; @@ -363,7 +366,7 @@ typedef struct SDataBase_S { bool drop; // 0: use exists, 1: if exists, drop then new create SDbCfg dbCfg; uint64_t superTblCount; - SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; + SSuperTable* superTbls; } SDataBase; typedef struct SDbs_S { @@ -382,12 +385,11 @@ typedef struct SDbs_S { uint32_t threadCount; uint32_t threadCountForCreateTbl; uint32_t dbCount; - SDataBase db[MAX_DB_COUNT]; - // statistics uint64_t totalInsertRows; uint64_t totalAffectedRows; + SDataBase* db; } SDbs; typedef struct SpecifiedQueryInfo_S { @@ -501,6 +503,7 @@ typedef struct SThreadInfo_S { uint64_t querySeq; // sequence number of sql command TAOS_SUB* tsub; + int sockfd; } threadInfo; #ifdef WINDOWS @@ -580,8 +583,7 @@ static void prompt(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); -static int postProceSql(char *host, struct sockaddr_in *pServAddr, - uint16_t port, char* sqlstr, threadInfo *pThreadInfo); +static int postProceSql(char *host, uint16_t port, char* sqlstr, threadInfo *pThreadInfo); static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, int disorderRange); static bool getInfoFromJsonFile(char* file); @@ -590,12 +592,12 @@ static int regexMatch(const char *s, const char *reg, int cflags); /* ************ Global variables ************ */ -int32_t g_randint[MAX_PREPARED_RAND]; -uint32_t g_randuint[MAX_PREPARED_RAND]; -int64_t g_randbigint[MAX_PREPARED_RAND]; -uint64_t g_randubigint[MAX_PREPARED_RAND]; -float g_randfloat[MAX_PREPARED_RAND]; -double g_randdouble[MAX_PREPARED_RAND]; +int32_t* g_randint; +uint32_t* g_randuint; +int64_t* g_randbigint; +uint64_t* g_randubigint; +float* g_randfloat; +double* g_randdouble; char *g_randbool_buff = NULL; char *g_randint_buff = NULL; @@ -631,6 +633,8 @@ SArguments g_args = { "tqueue", // password #elif (_TD_PRO_ == true) "prodb", // password +#elif (_TD_KH_ == true) + "khroot", // password #else "taosdata", // password #endif @@ -662,6 +666,7 @@ SArguments g_args = { 0, // insert_interval DEFAULT_TIMESTAMP_STEP, // timestamp_step 1, // query_times + 10000, // prepared_rand DEFAULT_INTERLACE_ROWS, // interlaceRows; 30000, // reqPerReq (1024*1024), // max_sql_len @@ -763,29 +768,12 @@ static void printHelp() { "The meta file to the execution procedure."); printf("%s%s%s%s\n", indent, "-u, --user=USER", "\t\t", "The user name to use when connecting to the server."); -#ifdef _TD_POWER_ - printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", - "The password to use when connecting to the server. By default is 'powerdb'"); - printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", - "Configuration directory. By default is '/etc/power/'."); -#elif (_TD_TQ_ == true) - printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", - "The password to use when connecting to the server. By default is 'tqueue'"); - printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", - "Configuration directory. By default is '/etc/tq/'."); -#elif (_TD_PRO_ == true) - printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", - "The password to use when connecting to the server. By default is 'prodb'"); - printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", - "Configuration directory. By default is '/etc/ProDB/'."); -#else printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", "The password to use when connecting to the server."); printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", "Configuration directory."); -#endif printf("%s%s%s%s\n", indent, "-h, --host=HOST", "\t\t", - "TDengine server FQDN to connect. The default host is localhost."); + "Server FQDN to connect. The default host is localhost."); printf("%s%s%s%s\n", indent, "-P, --port=PORT", "\t\t", "The TCP/IP port number to use for the connection."); printf("%s%s%s%s\n", indent, "-I, --interface=INTERFACE", "\t", @@ -796,6 +784,8 @@ static void printHelp() { "Set the replica parameters of the database, By default use 1, min: 1, max: 3."); printf("%s%s%s%s\n", indent, "-m, --table-prefix=TABLEPREFIX", "\t", "Table prefix name. By default use 'd'."); + printf("%s%s%s%s\n", indent, "-E, --escape-character", "\t", + "Use escape character for Both Stable and normmal table name"); printf("%s%s%s%s\n", indent, "-s, --sql-file=FILE", "\t\t", "The select sql file."); printf("%s%s%s%s\n", indent, "-N, --normal-table", "\t\t", "Use normal table flag."); @@ -851,10 +841,17 @@ static void printHelp() { /* printf("%s%s%s%s\n", indent, "-D", indent, "Delete database if exists. 0: no, 1: yes, default is 1"); */ +#ifdef _TD_KH_ + printf("\nMandatory or optional arguments to long options are also mandatory or optional\n\ +for any corresponding short options.\n\ +\n\ +Report bugs to .\n"); +#else printf("\nMandatory or optional arguments to long options are also mandatory or optional\n\ for any corresponding short options.\n\ \n\ Report bugs to .\n"); +#endif } static bool isStringNumber(char *input) @@ -1298,7 +1295,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "S"); exit(EXIT_FAILURE); } - arguments->async_mode = atoi(argv[++i]); + arguments->timestamp_step = atoi(argv[++i]); } else if (0 == strncmp(argv[i], "--time-step=", strlen("--time-step="))) { if (isStringNumber((char *)(argv[i] + strlen("--time-step=")))) { arguments->async_mode = atoi((char *)(argv[i]+strlen("--time-step="))); @@ -1308,7 +1305,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } } else if (0 == strncmp(argv[i], "-S", strlen("-S"))) { if (isStringNumber((char *)(argv[i] + strlen("-S")))) { - arguments->async_mode = atoi((char *)(argv[i]+strlen("-S"))); + arguments->timestamp_step = atoi((char *)(argv[i]+strlen("-S"))); } else { errorPrintReqArg2(argv[0], "-S"); exit(EXIT_FAILURE); @@ -1321,7 +1318,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--time-step"); exit(EXIT_FAILURE); } - arguments->async_mode = atoi(argv[++i]); + arguments->timestamp_step = atoi(argv[++i]); } else { errorUnrecognized(argv[0], argv[i]); exit(EXIT_FAILURE); @@ -1651,7 +1648,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->data_type[1] = TSDB_DATA_TYPE_NULL; } else { // more than one col - int index = 0; + int tdm_index = 0; g_dupstr = strdup(dataType); char *running = g_dupstr; char *token = strsep(&running, ","); @@ -1676,45 +1673,45 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } if (0 == strcasecmp(token, "INT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_INT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_INT; } else if (0 == strcasecmp(token, "FLOAT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_FLOAT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_FLOAT; } else if (0 == strcasecmp(token, "SMALLINT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_SMALLINT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_SMALLINT; } else if (0 == strcasecmp(token, "BIGINT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_BIGINT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_BIGINT; } else if (0 == strcasecmp(token, "DOUBLE")) { - arguments->data_type[index] = TSDB_DATA_TYPE_DOUBLE; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_DOUBLE; } else if (0 == strcasecmp(token, "TINYINT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_TINYINT; - } else if (1 == regexMatch(token, "^BINARY(\\([1-9][0-9]*\\))?$", REG_ICASE | + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_TINYINT; + } else if (1 == regexMatch(token, "^BINARY(\\([1-9][0-9]*\\))?$", REG_ICASE | REG_EXTENDED)) { - arguments->data_type[index] = TSDB_DATA_TYPE_BINARY; - } else if (1 == regexMatch(token, "^NCHAR(\\([1-9][0-9]*\\))?$", REG_ICASE | + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_BINARY; + } else if (1 == regexMatch(token, "^NCHAR(\\([1-9][0-9]*\\))?$", REG_ICASE | REG_EXTENDED)) { - arguments->data_type[index] = TSDB_DATA_TYPE_NCHAR; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_NCHAR; } else if (0 == strcasecmp(token, "BOOL")) { - arguments->data_type[index] = TSDB_DATA_TYPE_BOOL; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_BOOL; } else if (0 == strcasecmp(token, "TIMESTAMP")) { - arguments->data_type[index] = TSDB_DATA_TYPE_TIMESTAMP; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_TIMESTAMP; } else if (0 == strcasecmp(token, "UTINYINT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_UTINYINT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_UTINYINT; } else if (0 == strcasecmp(token, "USMALLINT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_USMALLINT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_USMALLINT; } else if (0 == strcasecmp(token, "UINT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_UINT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_UINT; } else if (0 == strcasecmp(token, "UBIGINT")) { - arguments->data_type[index] = TSDB_DATA_TYPE_UBIGINT; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_UBIGINT; } else { - arguments->data_type[index] = TSDB_DATA_TYPE_NULL; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_NULL; } - arguments->dataType[index] = token; - index ++; + arguments->dataType[tdm_index] = token; + tdm_index ++; token = strsep(&running, ","); - if (index >= MAX_NUM_COLUMNS) break; + if (tdm_index >= MAX_NUM_COLUMNS) break; } - arguments->dataType[index] = NULL; - arguments->data_type[index] = TSDB_DATA_TYPE_NULL; + arguments->dataType[tdm_index] = NULL; + arguments->data_type[tdm_index] = TSDB_DATA_TYPE_NULL; } } else if ((0 == strncmp(argv[i], "-w", strlen("-w"))) || (0 == strncmp(argv[i], "--binwidth", strlen("--binwidth")))) { @@ -2097,7 +2094,7 @@ static void tmfclose(FILE *fp) { } } -static void tmfree(char *buf) { +static void tmfree(void *buf) { if (NULL != buf) { free(buf); buf = NULL; @@ -2205,7 +2202,7 @@ static void selectAndGetResult( } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { int retCode = postProceSql( - g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, + g_queryInfo.host, g_queryInfo.port, command, pThreadInfo); if (0 != retCode) { @@ -2221,157 +2218,157 @@ static void selectAndGetResult( static char *rand_bool_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randbool_buff + ((cursor % MAX_PREPARED_RAND) * BOOL_BUFF_LEN); + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randbool_buff + ((cursor % g_args.prepared_rand) * BOOL_BUFF_LEN); } static int32_t rand_bool() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randint[cursor % MAX_PREPARED_RAND] % 2; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randint[cursor % g_args.prepared_rand] % 2; } static char *rand_tinyint_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_randtinyint_buff + - ((cursor % MAX_PREPARED_RAND) * TINYINT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * TINYINT_BUFF_LEN); } static int32_t rand_tinyint() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randint[cursor % MAX_PREPARED_RAND] % 128; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randint[cursor % g_args.prepared_rand] % 128; } static char *rand_utinyint_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_randutinyint_buff + - ((cursor % MAX_PREPARED_RAND) * TINYINT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * TINYINT_BUFF_LEN); } static int32_t rand_utinyint() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randuint[cursor % MAX_PREPARED_RAND] % 255; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randuint[cursor % g_args.prepared_rand] % 255; } static char *rand_smallint_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_randsmallint_buff + - ((cursor % MAX_PREPARED_RAND) * SMALLINT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * SMALLINT_BUFF_LEN); } static int32_t rand_smallint() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randint[cursor % MAX_PREPARED_RAND] % 32768; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randint[cursor % g_args.prepared_rand] % 32768; } static char *rand_usmallint_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_randusmallint_buff + - ((cursor % MAX_PREPARED_RAND) * SMALLINT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * SMALLINT_BUFF_LEN); } static int32_t rand_usmallint() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randuint[cursor % MAX_PREPARED_RAND] % 65535; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randuint[cursor % g_args.prepared_rand] % 65535; } static char *rand_int_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randint_buff + ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN); + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randint_buff + ((cursor % g_args.prepared_rand) * INT_BUFF_LEN); } static int32_t rand_int() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randint[cursor % MAX_PREPARED_RAND]; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randint[cursor % g_args.prepared_rand]; } static char *rand_uint_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randuint_buff + ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN); + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randuint_buff + ((cursor % g_args.prepared_rand) * INT_BUFF_LEN); } static int32_t rand_uint() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randuint[cursor % MAX_PREPARED_RAND]; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randuint[cursor % g_args.prepared_rand]; } static char *rand_bigint_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_randbigint_buff + - ((cursor % MAX_PREPARED_RAND) * BIGINT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * BIGINT_BUFF_LEN); } static int64_t rand_bigint() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randbigint[cursor % MAX_PREPARED_RAND]; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randbigint[cursor % g_args.prepared_rand]; } static char *rand_ubigint_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_randubigint_buff + - ((cursor % MAX_PREPARED_RAND) * BIGINT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * BIGINT_BUFF_LEN); } static int64_t rand_ubigint() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randubigint[cursor % MAX_PREPARED_RAND]; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randubigint[cursor % g_args.prepared_rand]; } static char *rand_float_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randfloat_buff + ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN); + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randfloat_buff + ((cursor % g_args.prepared_rand) * FLOAT_BUFF_LEN); } @@ -2379,58 +2376,58 @@ static float rand_float() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randfloat[cursor % MAX_PREPARED_RAND]; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_randfloat[cursor % g_args.prepared_rand]; } static char *demo_current_float_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_rand_current_buff + - ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * FLOAT_BUFF_LEN); } static float UNUSED_FUNC demo_current_float() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return (float)(9.8 + 0.04 * (g_randint[cursor % MAX_PREPARED_RAND] % 10) - + g_randfloat[cursor % MAX_PREPARED_RAND]/1000000000); + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return (float)(9.8 + 0.04 * (g_randint[cursor % g_args.prepared_rand] % 10) + + g_randfloat[cursor % g_args.prepared_rand]/1000000000); } static char *demo_voltage_int_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_rand_voltage_buff + - ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN); + ((cursor % g_args.prepared_rand) * INT_BUFF_LEN); } static int32_t UNUSED_FUNC demo_voltage_int() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return 215 + g_randint[cursor % MAX_PREPARED_RAND] % 10; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return 215 + g_randint[cursor % g_args.prepared_rand] % 10; } static char *demo_phase_float_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_rand_phase_buff + ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN); + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return g_rand_phase_buff + ((cursor % g_args.prepared_rand) * FLOAT_BUFF_LEN); } static float UNUSED_FUNC demo_phase_float() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return (float)((115 + g_randint[cursor % MAX_PREPARED_RAND] % 10 - + g_randfloat[cursor % MAX_PREPARED_RAND]/1000000000)/360); + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; + return (float)((115 + g_randint[cursor % g_args.prepared_rand] % 10 + + g_randfloat[cursor % g_args.prepared_rand]/1000000000)/360); } #if 0 @@ -2469,7 +2466,7 @@ static char *rand_double_str() { static int cursor; cursor++; - if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + if (cursor > (g_args.prepared_rand - 1)) cursor = 0; return g_randdouble_buff + (cursor * DOUBLE_BUFF_LEN); } @@ -2477,42 +2474,54 @@ static double rand_double() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; + cursor = cursor % g_args.prepared_rand; return g_randdouble[cursor]; } static void init_rand_data() { - g_randint_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + g_randint_buff = calloc(1, INT_BUFF_LEN * g_args.prepared_rand); assert(g_randint_buff); - g_rand_voltage_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + g_rand_voltage_buff = calloc(1, INT_BUFF_LEN * g_args.prepared_rand); assert(g_rand_voltage_buff); - g_randbigint_buff = calloc(1, BIGINT_BUFF_LEN * MAX_PREPARED_RAND); + g_randbigint_buff = calloc(1, BIGINT_BUFF_LEN * g_args.prepared_rand); assert(g_randbigint_buff); - g_randsmallint_buff = calloc(1, SMALLINT_BUFF_LEN * MAX_PREPARED_RAND); + g_randsmallint_buff = calloc(1, SMALLINT_BUFF_LEN * g_args.prepared_rand); assert(g_randsmallint_buff); - g_randtinyint_buff = calloc(1, TINYINT_BUFF_LEN * MAX_PREPARED_RAND); + g_randtinyint_buff = calloc(1, TINYINT_BUFF_LEN * g_args.prepared_rand); assert(g_randtinyint_buff); - g_randbool_buff = calloc(1, BOOL_BUFF_LEN * MAX_PREPARED_RAND); + g_randbool_buff = calloc(1, BOOL_BUFF_LEN * g_args.prepared_rand); assert(g_randbool_buff); - g_randfloat_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + g_randfloat_buff = calloc(1, FLOAT_BUFF_LEN * g_args.prepared_rand); assert(g_randfloat_buff); - g_rand_current_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + g_rand_current_buff = calloc(1, FLOAT_BUFF_LEN * g_args.prepared_rand); assert(g_rand_current_buff); - g_rand_phase_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + g_rand_phase_buff = calloc(1, FLOAT_BUFF_LEN * g_args.prepared_rand); assert(g_rand_phase_buff); - g_randdouble_buff = calloc(1, DOUBLE_BUFF_LEN * MAX_PREPARED_RAND); + g_randdouble_buff = calloc(1, DOUBLE_BUFF_LEN * g_args.prepared_rand); assert(g_randdouble_buff); - g_randuint_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + g_randuint_buff = calloc(1, INT_BUFF_LEN * g_args.prepared_rand); assert(g_randuint_buff); - g_randutinyint_buff = calloc(1, TINYINT_BUFF_LEN * MAX_PREPARED_RAND); + g_randutinyint_buff = calloc(1, TINYINT_BUFF_LEN * g_args.prepared_rand); assert(g_randutinyint_buff); - g_randusmallint_buff = calloc(1, SMALLINT_BUFF_LEN * MAX_PREPARED_RAND); + g_randusmallint_buff = calloc(1, SMALLINT_BUFF_LEN * g_args.prepared_rand); assert(g_randusmallint_buff); - g_randubigint_buff = calloc(1, BIGINT_BUFF_LEN * MAX_PREPARED_RAND); + g_randubigint_buff = calloc(1, BIGINT_BUFF_LEN * g_args.prepared_rand); assert(g_randubigint_buff); - - for (int i = 0; i < MAX_PREPARED_RAND; i++) { + g_randint = calloc(1, sizeof(int32_t) * g_args.prepared_rand); + assert(g_randint); + g_randuint = calloc(1, sizeof(uint32_t) * g_args.prepared_rand); + assert(g_randuint); + g_randbigint = calloc(1, sizeof(int64_t) * g_args.prepared_rand); + assert(g_randbigint); + g_randubigint = calloc(1, sizeof(uint64_t) * g_args.prepared_rand); + assert(g_randubigint); + g_randfloat = calloc(1, sizeof(float) * g_args.prepared_rand); + assert(g_randfloat); + g_randdouble = calloc(1, sizeof(double) * g_args.prepared_rand); + assert(g_randdouble); + + for (int i = 0; i < g_args.prepared_rand; i++) { g_randint[i] = (int)(taosRandom() % RAND_MAX - (RAND_MAX >> 1)); g_randuint[i] = (int)(taosRandom()); sprintf(g_randint_buff + i * INT_BUFF_LEN, "%d", @@ -2755,6 +2764,8 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].sampleFormat); printf(" sampleFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFile); + printf(" useSampleTs: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].useSampleTs ? "yes (warning: disorderRange/disorderRatio is disabled)" : "no"); printf(" tagsFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].tagsFile); printf(" columnCount: \033[33m%d\033[0m\n ", @@ -2799,8 +2810,6 @@ static int printfInsertMeta() { printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_args.insertRows); } - - printf("\n"); } @@ -3293,7 +3302,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { } static void printfDbInfoForQueryToFile( - char* filename, SDbInfo* dbInfos, int index) { + char* filename, SDbInfo* dbInfos, int tdm_index) { if (filename[0] == 0) return; @@ -3304,7 +3313,7 @@ static void printfDbInfoForQueryToFile( return; } - fprintf(fp, "================ database[%d] ================\n", index); + fprintf(fp, "================ database[%d] ================\n", tdm_index); fprintf(fp, "name: %s\n", dbInfos->name); fprintf(fp, "created_time: %s\n", dbInfos->create_time); fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables); @@ -3384,7 +3393,7 @@ static void printfQuerySystemInfo(TAOS * taos) { free(dbInfos); } -static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, +static int postProceSql(char *host, uint16_t port, char* sqlstr, threadInfo *pThreadInfo) { char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; @@ -3416,35 +3425,18 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; - snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", + if (g_args.test_mode == INSERT_TEST) { + snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", g_Dbs.user, g_Dbs.password); + } else { + snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", + g_queryInfo.user, g_queryInfo.password); + } + size_t userpass_buf_len = strlen(userpass_buf); size_t encoded_len = 4 * ((userpass_buf_len +2) / 3); char base64_buf[INPUT_BUF_LEN]; -#ifdef WINDOWS - WSADATA wsaData; - WSAStartup(MAKEWORD(2, 2), &wsaData); - SOCKET sockfd; -#else - int sockfd; -#endif - sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) { -#ifdef WINDOWS - errorPrint( "Could not create socket : %d" , WSAGetLastError()); -#endif - debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd); - free(request_buf); - ERROR_EXIT("opening socket"); - } - - int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr)); - debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn); - if (retConn < 0) { - free(request_buf); - ERROR_EXIT("connecting"); - } memset(base64_buf, 0, INPUT_BUF_LEN); @@ -3484,9 +3476,9 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port sent = 0; do { #ifdef WINDOWS - bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0); + bytes = send(pThreadInfo->sockfd, request_buf + sent, req_str_len - sent, 0); #else - bytes = write(sockfd, request_buf + sent, req_str_len - sent); + bytes = write(pThreadInfo->sockfd, request_buf + sent, req_str_len - sent); #endif if (bytes < 0) ERROR_EXIT("writing message to socket"); @@ -3498,12 +3490,18 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port memset(response_buf, 0, RESP_BUF_LEN); resp_len = sizeof(response_buf) - 1; received = 0; + + char resEncodingChunk[] = "Encoding: chunked"; + char resHttp[] = "HTTP/1.1 "; + char resHttpOk[] = "HTTP/1.1 200 OK"; + do { #ifdef WINDOWS - bytes = recv(sockfd, response_buf + received, resp_len - received, 0); + bytes = recv(pThreadInfo->sockfd, response_buf + received, resp_len - received, 0); #else - bytes = read(sockfd, response_buf + received, resp_len - received); + bytes = read(pThreadInfo->sockfd, response_buf + received, resp_len - received); #endif + verbosePrint("%s() LN%d: bytes:%d\n", __func__, __LINE__, bytes); if (bytes < 0) { free(request_buf); ERROR_EXIT("reading response from socket"); @@ -3511,6 +3509,22 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port if (bytes == 0) break; received += bytes; + + verbosePrint("%s() LN%d: received:%d resp_len:%d, response_buf:\n%s\n", + __func__, __LINE__, received, resp_len, response_buf); + + response_buf[RESP_BUF_LEN - 1] = '\0'; + if (strlen(response_buf)) { + if (((NULL == strstr(response_buf, resEncodingChunk)) + && (NULL != strstr(response_buf, resHttp))) + || ((NULL != strstr(response_buf, resHttpOk)) + && (NULL != strstr(response_buf, "\"status\":")))) { + debugPrint( + "%s() LN%d: received:%d resp_len:%d, response_buf:\n%s\n", + __func__, __LINE__, received, resp_len, response_buf); + break; + } + } } while(received < resp_len); if (received == resp_len) { @@ -3518,21 +3532,18 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port ERROR_EXIT("storing complete response from socket"); } - response_buf[RESP_BUF_LEN - 1] = '\0'; - printf("Response:\n%s\n", response_buf); - if (strlen(pThreadInfo->filePath) > 0) { appendResultBufToFile(response_buf, pThreadInfo); } free(request_buf); -#ifdef WINDOWS - closesocket(sockfd); - WSACleanup(); -#else - close(sockfd); -#endif + response_buf[RESP_BUF_LEN - 1] = '\0'; + if (NULL == strstr(response_buf, resHttpOk)) { + errorPrint("%s() LN%d, Response:\n%s\n", + __func__, __LINE__, response_buf); + return -1; + } return 0; } @@ -3790,8 +3801,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, limit, offset); //get all child table name use cmd: select tbname from superTblName; - snprintf(command, 1024, "select tbname from %s.%s %s", - dbName, stbName, limitBuf); + snprintf(command, 1024, "select tbname from %s.%s %s", dbName, stbName, limitBuf); res = taos_query(taos, command); int32_t code = taos_errno(res); @@ -3972,21 +3982,21 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - + if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - "INT", strlen("INT")) && + "INT", strlen("INT")) && strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_INT; } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - "TINYINT", strlen("TINYINT")) && + "TINYINT", strlen("TINYINT")) && strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_TINYINT; } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - "SMALLINT", strlen("SMALLINT")) && + "SMALLINT", strlen("SMALLINT")) && strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_SMALLINT; } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - "BIGINT", strlen("BIGINT")) && + "BIGINT", strlen("BIGINT")) && strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_BIGINT; } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], @@ -4040,7 +4050,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], min(NOTE_BUFF_LEN, fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1); - + if (strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { tstrncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], @@ -4319,9 +4329,10 @@ static int createSuperTable( superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow; + snprintf(command, BUFFER_SIZE, - "CREATE TABLE IF NOT EXISTS %s.%s (ts TIMESTAMP%s) TAGS %s", - dbName, superTbl->stbName, cols, tags); + "CREATE TABLE IF NOT EXISTS %s.%s (ts TIMESTAMP%s) TAGS %s", + dbName, superTbl->stbName, cols, tags); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { errorPrint2("create supertable %s failed!\n\n", superTbl->stbName); @@ -4567,7 +4578,6 @@ static void* createTable(void *sarg) return NULL; } pThreadInfo->tables_created += batchNum; - uint64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n", @@ -4581,8 +4591,8 @@ static void* createTable(void *sarg) NO_INSERT_TYPE, false)) { errorPrint2("queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); } + pThreadInfo->tables_created += batchNum; } - free(pThreadInfo->buffer); return NULL; } @@ -4789,6 +4799,23 @@ static int readTagFromCsvFileToMem(SSuperTable * stbInfo) { return 0; } +static void getAndSetRowsFromCsvFile(SSuperTable *stbInfo) { + FILE *fp = fopen(stbInfo->sampleFile, "r"); + int line_count = 0; + if (fp == NULL) { + errorPrint("Failed to open sample file: %s, reason:%s\n", + stbInfo->sampleFile, strerror(errno)); + exit(EXIT_FAILURE); + } + char *buf = calloc(1, stbInfo->maxSqlLen); + while (fgets(buf, stbInfo->maxSqlLen, fp)) { + line_count++; + } + fclose(fp); + tmfree(buf); + stbInfo->insertRows = line_count; +} + /* Read 10000 lines at most. If more than 10000 lines, continue to read after using */ @@ -4872,7 +4899,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } int count = 1; - int index = 0; + int tdm_index = 0; StrColumn columnCase; //superTbls->columnCount = columnSize; @@ -4915,42 +4942,48 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } for (int n = 0; n < count; ++n) { - tstrncpy(superTbls->columns[index].dataType, + tstrncpy(superTbls->columns[tdm_index].dataType, columnCase.dataType, min(DATATYPE_BUFF_LEN, strlen(columnCase.dataType) + 1)); - superTbls->columns[index].dataLen = columnCase.dataLen; - index++; + superTbls->columns[tdm_index].dataLen = columnCase.dataLen; + tdm_index++; } } - if ((index + 1 /* ts */) > MAX_NUM_COLUMNS) { + if ((tdm_index + 1 /* ts */) > MAX_NUM_COLUMNS) { errorPrint("failed to read json, column size overflow, allowed max column size is %d\n", MAX_NUM_COLUMNS); goto PARSE_OVER; } - superTbls->columnCount = index; + superTbls->columnCount = tdm_index; for (int c = 0; c < superTbls->columnCount; c++) { if (0 == strncasecmp(superTbls->columns[c].dataType, "INT", strlen("INT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_INT; + superTbls->columns[c].dataLen = sizeof(int); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "TINYINT", strlen("TINYINT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_TINYINT; + superTbls->columns[c].dataLen = sizeof(char); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "SMALLINT", strlen("SMALLINT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_SMALLINT; + superTbls->columns[c].dataLen = sizeof(int16_t); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "BIGINT", strlen("BIGINT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_BIGINT; + superTbls->columns[c].dataLen = sizeof(int64_t); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "FLOAT", strlen("FLOAT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_FLOAT; + superTbls->columns[c].dataLen = sizeof(float); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "DOUBLE", strlen("DOUBLE"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_DOUBLE; + superTbls->columns[c].dataLen = sizeof(double); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "BINARY", strlen("BINARY"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_BINARY; @@ -4960,28 +4993,34 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } else if (0 == strncasecmp(superTbls->columns[c].dataType, "BOOL", strlen("BOOL"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_BOOL; + superTbls->columns[c].dataLen = sizeof(char); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "TIMESTAMP", strlen("TIMESTAMP"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_TIMESTAMP; + superTbls->columns[c].dataLen = sizeof(int64_t); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "UTINYINT", strlen("UTINYINT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_UTINYINT; + superTbls->columns[c].dataLen = sizeof(char); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "USMALLINT", strlen("USMALLINT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_USMALLINT; + superTbls->columns[c].dataLen = sizeof(uint16_t); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "UINT", strlen("UINT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_UINT; + superTbls->columns[c].dataLen = sizeof(uint32_t); } else if (0 == strncasecmp(superTbls->columns[c].dataType, "UBIGINT", strlen("UBIGINT"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_UBIGINT; + superTbls->columns[c].dataLen = sizeof(uint64_t); } else { superTbls->columns[c].data_type = TSDB_DATA_TYPE_NULL; } } count = 1; - index = 0; + tdm_index = 0; // tags cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); if (!tags || tags->type != cJSON_Array) { @@ -5034,20 +5073,20 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } for (int n = 0; n < count; ++n) { - tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, + tstrncpy(superTbls->tags[tdm_index].dataType, columnCase.dataType, min(DATATYPE_BUFF_LEN, strlen(columnCase.dataType) + 1)); - superTbls->tags[index].dataLen = columnCase.dataLen; - index++; + superTbls->tags[tdm_index].dataLen = columnCase.dataLen; + tdm_index++; } } - if (index > TSDB_MAX_TAGS) { + if (tdm_index > TSDB_MAX_TAGS) { errorPrint("failed to read json, tags size overflow, allowed max tag count is %d\n", TSDB_MAX_TAGS); goto PARSE_OVER; } - superTbls->tagCount = index; + superTbls->tagCount = tdm_index; for (int t = 0; t < superTbls->tagCount; t++) { if (0 == strncasecmp(superTbls->tags[t].dataType, @@ -5242,6 +5281,22 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + cJSON* prepareRand = cJSON_GetObjectItem(root, "prepared_rand"); + if (prepareRand && prepareRand->type == cJSON_Number) { + if (prepareRand->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, prepared_rand input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_args.prepared_rand = prepareRand->valueint; + } else if (!prepareRand) { + g_args.prepared_rand = 10000; + } else { + errorPrint("%s() LN%d, failed to read json, prepared_rand not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, if (answerPrompt && answerPrompt->type == cJSON_String @@ -5283,7 +5338,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { MAX_DB_COUNT); goto PARSE_OVER; } - + g_Dbs.db = calloc(1, sizeof(SDataBase)*dbSize); + assert(g_Dbs.db); g_Dbs.dbCount = dbSize; for (int i = 0; i < dbSize; ++i) { cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); @@ -5459,10 +5515,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); - if (fsync && fsync->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.fsync = fsync->valueint; - } else if (!fsync) { + cJSON* kit_fsync= cJSON_GetObjectItem(dbinfo, "fsync"); + if (kit_fsync && kit_fsync->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.fsync = kit_fsync->valueint; + } else if (!kit_fsync) { g_Dbs.db[i].dbCfg.fsync = -1; } else { errorPrint("%s", "failed to read json, fsync input mistake\n"); @@ -5483,7 +5539,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { MAX_SUPER_TABLE_COUNT); goto PARSE_OVER; } - + g_Dbs.db[i].superTbls = calloc(1, stbSize * sizeof(SSuperTable)); + assert(g_Dbs.db[i].superTbls); g_Dbs.db[i].superTblCount = stbSize; for (int j = 0; j < stbSize; ++j) { cJSON* stbInfo = cJSON_GetArrayItem(stables, j); @@ -5682,6 +5739,23 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + cJSON *useSampleTs = cJSON_GetObjectItem(stbInfo, "use_sample_ts"); + if (useSampleTs && useSampleTs->type == cJSON_String + && useSampleTs->valuestring != NULL) { + if (0 == strncasecmp(useSampleTs->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].useSampleTs = true; + } else if (0 == strncasecmp(useSampleTs->valuestring, "no", 2)){ + g_Dbs.db[i].superTbls[j].useSampleTs = false; + } else { + g_Dbs.db[i].superTbls[j].useSampleTs = false; + } + } else if (!useSampleTs) { + g_Dbs.db[i].superTbls[j].useSampleTs = false; + } else { + errorPrint("%s", "failed to read json, use_sample_ts not found\n"); + goto PARSE_OVER; + } + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); if ((tagsFile && tagsFile->type == cJSON_String) && (tagsFile->valuestring != NULL)) { @@ -6339,9 +6413,12 @@ static bool getInfoFromJsonFile(char* file) { } if (INSERT_TEST == g_args.test_mode) { + memset(&g_Dbs, 0, sizeof(SDbs)); + g_Dbs.use_metric = g_args.use_metric; ret = getMetaFromInsertJsonFile(root); } else if ((QUERY_TEST == g_args.test_mode) || (SUBSCRIBE_TEST == g_args.test_mode)) { + memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); ret = getMetaFromQueryJsonFile(root); } else { errorPrint("%s", @@ -6406,8 +6483,9 @@ static void postFreeResource() { g_Dbs.db[i].superTbls[j].childTblName = NULL; } } + tmfree(g_Dbs.db[i].superTbls); } - + tmfree(g_Dbs.db); tmfree(g_randbool_buff); tmfree(g_randint_buff); tmfree(g_rand_voltage_buff); @@ -6417,6 +6495,17 @@ static void postFreeResource() { tmfree(g_randfloat_buff); tmfree(g_rand_current_buff); tmfree(g_rand_phase_buff); + tmfree(g_randdouble_buff); + tmfree(g_randuint_buff); + tmfree(g_randutinyint_buff); + tmfree(g_randusmallint_buff); + tmfree(g_randubigint_buff); + tmfree(g_randint); + tmfree(g_randuint); + tmfree(g_randbigint); + tmfree(g_randubigint); + tmfree(g_randfloat); + tmfree(g_randdouble); tmfree(g_sampleDataBuf); @@ -6430,6 +6519,7 @@ static void postFreeResource() { } } tmfree(g_sampleBindBatchArray); + #endif } @@ -6442,13 +6532,20 @@ static int getRowDataFromSample( } int dataLen = 0; - - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + if(stbInfo->useSampleTs) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + "(%s", + stbInfo->sampleDataBuf + + stbInfo->lenOfOneRow * (*sampleUsePos)); + } else { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%s", stbInfo->sampleDataBuf + stbInfo->lenOfOneRow * (*sampleUsePos)); + } + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); (*sampleUsePos)++; @@ -6520,7 +6617,7 @@ static int64_t generateStbRowData( tmpLen = strlen(tmp); tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, BIGINT_BUFF_LEN)); break; - + case TSDB_DATA_TYPE_UBIGINT: tmp = rand_ubigint_str(); tmpLen = strlen(tmp); @@ -6881,6 +6978,9 @@ static int prepareSampleForStb(SSuperTable *stbInfo) { int ret; if (0 == strncasecmp(stbInfo->dataSource, "sample", strlen("sample"))) { + if(stbInfo->useSampleTs) { + getAndSetRowsFromCsvFile(stbInfo); + } ret = generateSampleFromCsvForStb(stbInfo); } else { ret = generateSampleFromRandForStb(stbInfo); @@ -6931,7 +7031,7 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); - if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, + if (0 != postProceSql(g_Dbs.host, g_Dbs.port, pThreadInfo->buffer, pThreadInfo)) { affectedRows = -1; printf("========restful return fail, threadID[%d]\n", @@ -6982,12 +7082,11 @@ static void getTableName(char *pTblName, stbInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN); } } else { - snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"", - stbInfo->childTblPrefix, tableSeq); + snprintf(pTblName, TSDB_TABLE_NAME_LEN, + "%s%"PRIu64"", stbInfo->childTblPrefix, tableSeq); } } else { - snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"", - g_args.tb_prefix, tableSeq); + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"", g_args.tb_prefix, tableSeq); } } @@ -7326,7 +7425,7 @@ static int64_t generateInterlaceDataWithoutStb( } static int32_t prepareStmtBindArrayByType( - TAOS_BIND *bind, + TAOS_BIND *kit_bind, char data_type, int32_t dataLen, int32_t timePrec, char *value) @@ -7353,20 +7452,20 @@ static int32_t prepareStmtBindArrayByType( } char *bind_binary; - bind->buffer_type = TSDB_DATA_TYPE_BINARY; + kit_bind->buffer_type = TSDB_DATA_TYPE_BINARY; if (value) { bind_binary = calloc(1, strlen(value) + 1); strncpy(bind_binary, value, strlen(value)); - bind->buffer_length = strlen(bind_binary); + kit_bind->buffer_length = strlen(bind_binary); } else { bind_binary = calloc(1, dataLen + 1); rand_string(bind_binary, dataLen); - bind->buffer_length = dataLen; + kit_bind->buffer_length = dataLen; } - bind->length = &bind->buffer_length; - bind->buffer = bind_binary; - bind->is_null = NULL; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->buffer = bind_binary; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_NCHAR: @@ -7377,7 +7476,7 @@ static int32_t prepareStmtBindArrayByType( } char *bind_nchar; - bind->buffer_type = TSDB_DATA_TYPE_NCHAR; + kit_bind->buffer_type = TSDB_DATA_TYPE_NCHAR; if (value) { bind_nchar = calloc(1, strlen(value) + 1); strncpy(bind_nchar, value, strlen(value)); @@ -7386,10 +7485,10 @@ static int32_t prepareStmtBindArrayByType( rand_string(bind_nchar, dataLen); } - bind->buffer_length = strlen(bind_nchar); - bind->buffer = bind_nchar; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_length = strlen(bind_nchar); + kit_bind->buffer = bind_nchar; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_INT: @@ -7401,13 +7500,13 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_int = rand_int(); } - bind->buffer_type = TSDB_DATA_TYPE_INT; - bind->buffer_length = sizeof(int32_t); - bind->buffer = bind_int; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_INT; + kit_bind->buffer_length = sizeof(int32_t); + kit_bind->buffer = bind_int; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; - + case TSDB_DATA_TYPE_UINT: bind_uint = malloc(sizeof(uint32_t)); assert(bind_uint); @@ -7417,11 +7516,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_uint = rand_int(); } - bind->buffer_type = TSDB_DATA_TYPE_UINT; - bind->buffer_length = sizeof(uint32_t); - bind->buffer = bind_uint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_UINT; + kit_bind->buffer_length = sizeof(uint32_t); + kit_bind->buffer = bind_uint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_BIGINT: @@ -7433,11 +7532,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_bigint = rand_bigint(); } - bind->buffer_type = TSDB_DATA_TYPE_BIGINT; - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_bigint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_BIGINT; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = bind_bigint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_UBIGINT: @@ -7449,11 +7548,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_ubigint = rand_bigint(); } - bind->buffer_type = TSDB_DATA_TYPE_UBIGINT; - bind->buffer_length = sizeof(uint64_t); - bind->buffer = bind_ubigint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_UBIGINT; + kit_bind->buffer_length = sizeof(uint64_t); + kit_bind->buffer = bind_ubigint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_FLOAT: @@ -7465,11 +7564,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_float = rand_float(); } - bind->buffer_type = TSDB_DATA_TYPE_FLOAT; - bind->buffer_length = sizeof(float); - bind->buffer = bind_float; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_FLOAT; + kit_bind->buffer_length = sizeof(float); + kit_bind->buffer = bind_float; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_DOUBLE: @@ -7481,11 +7580,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_double = rand_double(); } - bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; - bind->buffer_length = sizeof(double); - bind->buffer = bind_double; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; + kit_bind->buffer_length = sizeof(double); + kit_bind->buffer = bind_double; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_SMALLINT: @@ -7497,11 +7596,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_smallint = rand_smallint(); } - bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; - bind->buffer_length = sizeof(int16_t); - bind->buffer = bind_smallint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; + kit_bind->buffer_length = sizeof(int16_t); + kit_bind->buffer = bind_smallint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_USMALLINT: @@ -7513,11 +7612,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_usmallint = rand_smallint(); } - bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; - bind->buffer_length = sizeof(uint16_t); - bind->buffer = bind_usmallint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; + kit_bind->buffer_length = sizeof(uint16_t); + kit_bind->buffer = bind_usmallint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_TINYINT: @@ -7529,11 +7628,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_tinyint = rand_tinyint(); } - bind->buffer_type = TSDB_DATA_TYPE_TINYINT; - bind->buffer_length = sizeof(int8_t); - bind->buffer = bind_tinyint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_TINYINT; + kit_bind->buffer_length = sizeof(int8_t); + kit_bind->buffer = bind_tinyint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_UTINYINT: @@ -7545,11 +7644,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_utinyint = rand_tinyint(); } - bind->buffer_type = TSDB_DATA_TYPE_UTINYINT; - bind->buffer_length = sizeof(uint8_t); - bind->buffer = bind_utinyint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_UTINYINT; + kit_bind->buffer_length = sizeof(uint8_t); + kit_bind->buffer = bind_utinyint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_BOOL: @@ -7565,11 +7664,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_bool = rand_bool(); } - bind->buffer_type = TSDB_DATA_TYPE_BOOL; - bind->buffer_length = sizeof(int8_t); - bind->buffer = bind_bool; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_BOOL; + kit_bind->buffer_length = sizeof(int8_t); + kit_bind->buffer = bind_bool; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_TIMESTAMP: @@ -7600,11 +7699,11 @@ static int32_t prepareStmtBindArrayByType( } else { *bind_ts2 = rand_bigint(); } - bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_ts2; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = bind_ts2; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; break; case TSDB_DATA_TYPE_NULL: @@ -7619,7 +7718,7 @@ static int32_t prepareStmtBindArrayByType( } static int32_t prepareStmtBindArrayByTypeForRand( - TAOS_BIND *bind, + TAOS_BIND *kit_bind, char data_type, int32_t dataLen, int32_t timePrec, char **ptr, @@ -7648,20 +7747,20 @@ static int32_t prepareStmtBindArrayByTypeForRand( } char *bind_binary = (char *)*ptr; - bind->buffer_type = TSDB_DATA_TYPE_BINARY; + kit_bind->buffer_type = TSDB_DATA_TYPE_BINARY; if (value) { strncpy(bind_binary, value, strlen(value)); - bind->buffer_length = strlen(bind_binary); + kit_bind->buffer_length = strlen(bind_binary); } else { rand_string(bind_binary, dataLen); - bind->buffer_length = dataLen; + kit_bind->buffer_length = dataLen; } - bind->length = &bind->buffer_length; - bind->buffer = bind_binary; - bind->is_null = NULL; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->buffer = bind_binary; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_NCHAR: @@ -7672,19 +7771,19 @@ static int32_t prepareStmtBindArrayByTypeForRand( } char *bind_nchar = (char *)*ptr; - bind->buffer_type = TSDB_DATA_TYPE_NCHAR; + kit_bind->buffer_type = TSDB_DATA_TYPE_NCHAR; if (value) { strncpy(bind_nchar, value, strlen(value)); } else { rand_string(bind_nchar, dataLen); } - bind->buffer_length = strlen(bind_nchar); - bind->buffer = bind_nchar; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_length = strlen(bind_nchar); + kit_bind->buffer = bind_nchar; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_INT: @@ -7695,13 +7794,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_int = rand_int(); } - bind->buffer_type = TSDB_DATA_TYPE_INT; - bind->buffer_length = sizeof(int32_t); - bind->buffer = bind_int; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_INT; + kit_bind->buffer_length = sizeof(int32_t); + kit_bind->buffer = bind_int; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_UINT: @@ -7712,13 +7811,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_uint = rand_int(); } - bind->buffer_type = TSDB_DATA_TYPE_UINT; - bind->buffer_length = sizeof(uint32_t); - bind->buffer = bind_uint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_UINT; + kit_bind->buffer_length = sizeof(uint32_t); + kit_bind->buffer = bind_uint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_BIGINT: @@ -7729,13 +7828,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_bigint = rand_bigint(); } - bind->buffer_type = TSDB_DATA_TYPE_BIGINT; - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_bigint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_BIGINT; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = bind_bigint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_UBIGINT: @@ -7746,13 +7845,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_ubigint = rand_bigint(); } - bind->buffer_type = TSDB_DATA_TYPE_UBIGINT; - bind->buffer_length = sizeof(uint64_t); - bind->buffer = bind_ubigint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_UBIGINT; + kit_bind->buffer_length = sizeof(uint64_t); + kit_bind->buffer = bind_ubigint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_FLOAT: @@ -7763,13 +7862,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_float = rand_float(); } - bind->buffer_type = TSDB_DATA_TYPE_FLOAT; - bind->buffer_length = sizeof(float); - bind->buffer = bind_float; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_FLOAT; + kit_bind->buffer_length = sizeof(float); + kit_bind->buffer = bind_float; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_DOUBLE: @@ -7780,13 +7879,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_double = rand_double(); } - bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; - bind->buffer_length = sizeof(double); - bind->buffer = bind_double; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; + kit_bind->buffer_length = sizeof(double); + kit_bind->buffer = bind_double; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_SMALLINT: @@ -7797,13 +7896,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_smallint = rand_smallint(); } - bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; - bind->buffer_length = sizeof(int16_t); - bind->buffer = bind_smallint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; + kit_bind->buffer_length = sizeof(int16_t); + kit_bind->buffer = bind_smallint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_USMALLINT: @@ -7814,13 +7913,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_usmallint = rand_smallint(); } - bind->buffer_type = TSDB_DATA_TYPE_USMALLINT; - bind->buffer_length = sizeof(uint16_t); - bind->buffer = bind_usmallint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_USMALLINT; + kit_bind->buffer_length = sizeof(uint16_t); + kit_bind->buffer = bind_usmallint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_TINYINT: @@ -7831,13 +7930,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_tinyint = rand_tinyint(); } - bind->buffer_type = TSDB_DATA_TYPE_TINYINT; - bind->buffer_length = sizeof(int8_t); - bind->buffer = bind_tinyint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_TINYINT; + kit_bind->buffer_length = sizeof(int8_t); + kit_bind->buffer = bind_tinyint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_UTINYINT: @@ -7848,13 +7947,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_utinyint = rand_tinyint(); } - bind->buffer_type = TSDB_DATA_TYPE_UTINYINT; - bind->buffer_length = sizeof(uint8_t); - bind->buffer = bind_utinyint; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_UTINYINT; + kit_bind->buffer_length = sizeof(uint8_t); + kit_bind->buffer = bind_utinyint; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_BOOL: @@ -7869,13 +7968,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_bool = rand_bool(); } - bind->buffer_type = TSDB_DATA_TYPE_BOOL; - bind->buffer_length = sizeof(int8_t); - bind->buffer = bind_bool; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_BOOL; + kit_bind->buffer_length = sizeof(int8_t); + kit_bind->buffer = bind_bool; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; case TSDB_DATA_TYPE_TIMESTAMP: @@ -7904,13 +8003,13 @@ static int32_t prepareStmtBindArrayByTypeForRand( } else { *bind_ts2 = rand_bigint(); } - bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_ts2; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = bind_ts2; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - *ptr += bind->buffer_length; + *ptr += kit_bind->buffer_length; break; default: @@ -7950,11 +8049,11 @@ static int32_t prepareStmtWithoutStb( for (k = 0; k < batch;) { /* columnCount + 1 (ts) */ - TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0); + TAOS_BIND *kit_bind = (TAOS_BIND *)(bindArray + 0); int64_t *bind_ts = pThreadInfo->bind_ts; - bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + kit_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; if (g_args.disorderRatio) { *bind_ts = startTime + getTSRandTail( @@ -7964,16 +8063,16 @@ static int32_t prepareStmtWithoutStb( } else { *bind_ts = startTime + g_args.timestamp_step * k; } - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_ts; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = bind_ts; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; for (int i = 0; i < g_args.columnCount; i ++) { - bind = (TAOS_BIND *)((char *)bindArray + kit_bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); if ( -1 == prepareStmtBindArrayByType( - bind, + kit_bind, data_type[i], g_args.binwidth, pThreadInfo->time_precision, @@ -8037,15 +8136,15 @@ static int32_t prepareStbStmtBindRand( memset(data, 0, MAX_DATA_SIZE); char *ptr = data; - TAOS_BIND *bind; + TAOS_BIND *kit_bind; for (int i = 0; i < stbInfo->columnCount + 1; i ++) { - bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * i)); + kit_bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * i)); if (i == 0) { int64_t *bind_ts = ts; - bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + kit_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; if (stbInfo->disorderRatio) { *bind_ts = startTime + getTSRandTail( stbInfo->timeStampStep, recSeq, @@ -8054,14 +8153,14 @@ static int32_t prepareStbStmtBindRand( } else { *bind_ts = startTime + stbInfo->timeStampStep * recSeq; } - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_ts; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = bind_ts; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; - ptr += bind->buffer_length; + ptr += kit_bind->buffer_length; } else if ( -1 == prepareStmtBindArrayByTypeForRand( - bind, + kit_bind, stbInfo->columns[i-1].data_type, stbInfo->columns[i-1].dataLen, timePrec, @@ -8475,7 +8574,7 @@ static int parseSamplefileToStmtBatch( case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: tmpP = calloc(1, MAX_SAMPLES * - (((stbInfo)?stbInfo->columns[c].dataLen:g_args.binwidth))); + (((stbInfo)?stbInfo->columns[c].dataLen:g_args.binwidth) + 1)); assert(tmpP); *(uintptr_t*)(sampleBindBatchArray+ sizeof(uintptr_t*)*c) = (uintptr_t)tmpP; break; @@ -8507,22 +8606,22 @@ static int parseSamplefileToStmtBatch( + lenOfOneRow * i + cursor; int lengthOfRest = strlen(restStr); - int index = 0; - for (index = 0; index < lengthOfRest; index ++) { - if (restStr[index] == ',') { + int tdm_index = 0; + for (tdm_index = 0; tdm_index < lengthOfRest; tdm_index ++) { + if (restStr[tdm_index] == ',') { break; } } - char *tmpStr = calloc(1, index + 1); + char *tmpStr = calloc(1, tdm_index + 1); if (NULL == tmpStr) { errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, index + 1); + __func__, __LINE__, tdm_index + 1); return -1; } - strncpy(tmpStr, restStr, index); - cursor += index + 1; // skip ',' too + strncpy(tmpStr, restStr, tdm_index); + cursor += tdm_index + 1; // skip ',' too char *tmpP; switch(data_type) { @@ -8661,18 +8760,18 @@ static int parseSampleToStmt( return -1; } - TAOS_BIND *bind; + TAOS_BIND *kit_bind; int cursor = 0; for (int c = 0; c < columnCount + 1; c++) { - bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * c)); + kit_bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * c)); if (c == 0) { - bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - bind->buffer_length = sizeof(int64_t); - bind->buffer = NULL; //bind_ts; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = NULL; //bind_ts; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; } else { char data_type = (stbInfo)? stbInfo->columns[c-1].data_type: @@ -8684,25 +8783,25 @@ static int parseSampleToStmt( + lenOfOneRow * i + cursor; int lengthOfRest = strlen(restStr); - int index = 0; - for (index = 0; index < lengthOfRest; index ++) { - if (restStr[index] == ',') { + int tdm_index = 0; + for (tdm_index = 0; tdm_index < lengthOfRest; tdm_index ++) { + if (restStr[tdm_index] == ',') { break; } } - char *bindBuffer = calloc(1, index + 1); + char *bindBuffer = calloc(1, tdm_index + 1); if (bindBuffer == NULL) { errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, index + 1); + __func__, __LINE__, tdm_index + 1); return -1; } - strncpy(bindBuffer, restStr, index); - cursor += index + 1; // skip ',' too + strncpy(bindBuffer, restStr, tdm_index); + cursor += tdm_index + 1; // skip ',' too if (-1 == prepareStmtBindArrayByType( - bind, + kit_bind, data_type, dataLen, timePrec, @@ -8746,13 +8845,13 @@ static int32_t prepareStbStmtBindStartTime( char *bindArray, SSuperTable *stbInfo, int64_t startTime, int32_t recSeq) { - TAOS_BIND *bind; + TAOS_BIND *kit_bind; - bind = (TAOS_BIND *)bindArray; + kit_bind = (TAOS_BIND *)bindArray; int64_t *bind_ts = ts; - bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + kit_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; if (stbInfo->disorderRatio) { *bind_ts = startTime + getTSRandTail( stbInfo->timeStampStep, recSeq, @@ -8765,10 +8864,10 @@ static int32_t prepareStbStmtBindStartTime( verbosePrint("%s() LN%d, tableName: %s, bind_ts=%"PRId64"\n", __func__, __LINE__, tableName, *bind_ts); - bind->buffer_length = sizeof(int64_t); - bind->buffer = bind_ts; - bind->length = &bind->buffer_length; - bind->is_null = NULL; + kit_bind->buffer_length = sizeof(int64_t); + kit_bind->buffer = bind_ts; + kit_bind->length = &kit_bind->buffer_length; + kit_bind->is_null = NULL; return 0; } @@ -10231,7 +10330,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } int64_t ntables = 0; - uint64_t tableFrom; + uint64_t tableFrom = 0; if (stbInfo) { int64_t limit; @@ -10470,6 +10569,33 @@ static void startMultiThreadInsertData(int threads, char* db_name, pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); } */ + + if (g_args.iface == REST_IFACE || ((stbInfo) && (stbInfo->iface == REST_IFACE))) { +#ifdef WINDOWS + WSADATA wsaData; + WSAStartup(MAKEWORD(2, 2), &wsaData); + SOCKET sockfd; +#else + int sockfd; +#endif + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { +#ifdef WINDOWS + errorPrint( "Could not create socket : %d" , WSAGetLastError()); +#endif + debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd); + ERROR_EXIT("opening socket"); + } + + int retConn = connect(sockfd, (struct sockaddr *)&(g_Dbs.serv_addr), sizeof(struct sockaddr)); + debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn); + if (retConn < 0) { + ERROR_EXIT("connecting"); + } + pThreadInfo->sockfd = sockfd; + } + + tsem_init(&(pThreadInfo->lock_sem), 0, 0); if (ASYNC_MODE == g_Dbs.asyncMode) { pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); @@ -10507,6 +10633,14 @@ static void startMultiThreadInsertData(int threads, char* db_name, tmfree((char *)pThreadInfo->bind_ts_array); tmfree(pThreadInfo->bindParams); tmfree(pThreadInfo->is_null); + if (g_args.iface == REST_IFACE || ((stbInfo) && (stbInfo->iface == REST_IFACE))) { +#ifdef WINDOWS + closesocket(pThreadInfo->sockfd); + WSACleanup(); +#else + close(pThreadInfo->sockfd); +#endif + } #else if (pThreadInfo->sampleBindArray) { for (int k = 0; k < MAX_SAMPLES; k++) { @@ -10517,9 +10651,9 @@ static void startMultiThreadInsertData(int threads, char* db_name, pThreadInfo->stbInfo->columnCount: g_args.columnCount; for (int c = 1; c < columnCount + 1; c++) { - TAOS_BIND *bind = (TAOS_BIND *)((char *)tmp + (sizeof(TAOS_BIND) * c)); - if (bind) - tmfree(bind->buffer); + TAOS_BIND *kit_bind = (TAOS_BIND *)((char *)tmp + (sizeof(TAOS_BIND) * c)); + if (kit_bind) + tmfree(kit_bind->buffer); } tmfree((char *)tmp); } @@ -10829,8 +10963,6 @@ static int insertTestProcess() { assert(cmdBuffer); if(createDatabasesAndStables(cmdBuffer) != 0) { - if (g_fpOfInsertResult) - fclose(g_fpOfInsertResult); free(cmdBuffer); return -1; } @@ -10909,7 +11041,6 @@ static int insertTestProcess() { // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; //} //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount); - postFreeResource(); return 0; } @@ -11169,6 +11300,31 @@ static int queryTestProcess() { } } + if (0 == strncasecmp(g_queryInfo.queryMode, "rest", 4)) { +#ifdef WINDOWS + WSADATA wsaData; + WSAStartup(MAKEWORD(2, 2), &wsaData); + SOCKET sockfd; +#else + int sockfd; +#endif + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { +#ifdef WINDOWS + errorPrint( "Could not create socket : %d" , WSAGetLastError()); +#endif + debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd); + ERROR_EXIT("opening socket"); + } + + int retConn = connect(sockfd, (struct sockaddr *)&(g_queryInfo.serv_addr), + sizeof(struct sockaddr)); + debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn); + if (retConn < 0) { + ERROR_EXIT("connecting"); + } + pThreadInfo->sockfd = sockfd; + } pThreadInfo->taos = NULL;// workaround to use separate taos connection; pthread_create(pids + seq, NULL, specifiedTableQuery, @@ -11220,6 +11376,31 @@ static int queryTestProcess() { pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; tableFrom = pThreadInfo->end_table_to + 1; pThreadInfo->taos = NULL; // workaround to use separate taos connection; + if (0 == strncasecmp(g_queryInfo.queryMode, "rest", 4)) { +#ifdef WINDOWS + WSADATA wsaData; + WSAStartup(MAKEWORD(2, 2), &wsaData); + SOCKET sockfd; +#else + int sockfd; +#endif + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { +#ifdef WINDOWS + errorPrint( "Could not create socket : %d" , WSAGetLastError()); +#endif + debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd); + ERROR_EXIT("opening socket"); + } + + int retConn = connect(sockfd, (struct sockaddr *)&(g_queryInfo.serv_addr), + sizeof(struct sockaddr)); + debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn); + if (retConn < 0) { + ERROR_EXIT("connecting"); + } + pThreadInfo->sockfd = sockfd; + } pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo); } @@ -11232,6 +11413,15 @@ static int queryTestProcess() { for (int i = 0; i < nConcurrent; i++) { for (int j = 0; j < nSqlCount; j++) { pthread_join(pids[i * nSqlCount + j], NULL); + if (0 == strncasecmp(g_queryInfo.queryMode, "rest", 4)) { + threadInfo *pThreadInfo = infos + i * nSqlCount + j; +#ifdef WINDOWS + closesocket(pThreadInfo->sockfd); + WSACleanup(); +#else + close(pThreadInfo->sockfd); +#endif + } } } } @@ -11241,6 +11431,15 @@ static int queryTestProcess() { for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { pthread_join(pidsOfSub[i], NULL); + if (0 == strncasecmp(g_queryInfo.queryMode, "rest", 4)) { + threadInfo *pThreadInfo = infosOfSub + i; +#ifdef WINDOWS + closesocket(pThreadInfo->sockfd); + WSACleanup(); +#else + close(pThreadInfo->sockfd); +#endif + } } tmfree((char*)pidsOfSub); @@ -11743,29 +11942,6 @@ static int subscribeTestProcess() { return 0; } -static void initOfInsertMeta() { - memset(&g_Dbs, 0, sizeof(SDbs)); - - // set default values - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - g_Dbs.port = 6030; - tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN); - g_Dbs.threadCount = 2; - - g_Dbs.use_metric = g_args.use_metric; -} - -static void initOfQueryMeta() { - memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); - - // set default values - tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - g_queryInfo.port = 6030; - tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); - tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN); -} - static void setParaFromArg() { char type[20]; char length[20]; @@ -11798,7 +11974,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); g_Dbs.use_metric = g_args.use_metric; - + g_args.prepared_rand = min(g_args.insertRows, MAX_PREPARED_RAND); g_Dbs.aggr_func = g_args.aggr_func; char dataString[TSDB_MAX_BYTES_PER_ROW]; @@ -11857,7 +12033,35 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, type, min(DATATYPE_BUFF_LEN, strlen(type) + 1)); } else { - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.binwidth; + switch (g_Dbs.db[0].superTbls[0].columns[i].data_type){ + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: + g_Dbs.db[0].superTbls[0].columns[i].dataLen = sizeof(char); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + g_Dbs.db[0].superTbls[0].columns[i].dataLen = sizeof(int16_t); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + g_Dbs.db[0].superTbls[0].columns[i].dataLen = sizeof(int32_t); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + g_Dbs.db[0].superTbls[0].columns[i].dataLen = sizeof(int64_t); + break; + case TSDB_DATA_TYPE_FLOAT: + g_Dbs.db[0].superTbls[0].columns[i].dataLen = sizeof(float); + break; + case TSDB_DATA_TYPE_DOUBLE: + g_Dbs.db[0].superTbls[0].columns[i].dataLen = sizeof(double); + break; + default: + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.binwidth; + break; + } tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, dataType[i], min(DATATYPE_BUFF_LEN, strlen(dataType[i]) + 1)); } @@ -11915,7 +12119,6 @@ static int regexMatch(const char *s, const char *reg, int cflags) { printf("Regex match failed: %s\n", msgbuf); exit(EXIT_FAILURE); } - return 0; } @@ -12075,8 +12278,6 @@ int main(int argc, char *argv[]) { if (g_args.metaFile) { g_totalChildTables = 0; - initOfInsertMeta(); - initOfQueryMeta(); if (false == getInfoFromJsonFile(g_args.metaFile)) { printf("Failed to read %s\n", g_args.metaFile); @@ -12086,6 +12287,10 @@ int main(int argc, char *argv[]) { testMetaFile(); } else { memset(&g_Dbs, 0, sizeof(SDbs)); + g_Dbs.db = calloc(1, sizeof(SDataBase)); + assert(g_Dbs.db); + g_Dbs.db[0].superTbls = calloc(1, sizeof(SSuperTable)); + assert(g_Dbs.db[0].superTbls); setParaFromArg(); if (NULL != g_args.sqlFile) { @@ -12105,6 +12310,7 @@ int main(int argc, char *argv[]) { if (g_dupstr) free(g_dupstr); } + postFreeResource(); return 0; -} \ No newline at end of file +} diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 34d9956c642c5c91aa72c6a4386dc675ad48dc98..d593f1c4bb5d939432ebdea83b5f461f5201a52e 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -231,7 +231,12 @@ SDbInfo **g_dbInfos = NULL; TableInfo *g_tablesList = NULL; const char *argp_program_version = version; +#ifdef _TD_KH_ +const char *argp_program_bug_address = ""; +#else const char *argp_program_bug_address = ""; +#endif + /* Program documentation. */ static char doc[] = ""; @@ -252,22 +257,14 @@ static struct argp_option options[] = { // connection option {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, -#ifdef _TD_POWER_ - {"password", 'p', 0, 0, "User password to connect to server. Default is powerdb.", 0}, -#else {"password", 'p', 0, 0, "User password to connect to server. Default is taosdata.", 0}, -#endif {"port", 'P', "PORT", 0, "Port to connect", 0}, {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, // input/output file {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, -#ifdef _TD_POWER_ - {"config-dir", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, -#else {"config-dir", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, -#endif {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, @@ -364,6 +361,12 @@ struct arguments g_args = { "root", #ifdef _TD_POWER_ "powerdb", +#elif (_TD_TQ_ == true) + "tqueue", +#elif (_TD_PRO_ == true) + "prodb", +#elif (_TD_KH_ == true) + "khroot", #else "taosdata", #endif @@ -1972,7 +1975,7 @@ static int getTableDes( while ((row = taos_fetch_row(res)) != NULL) { tstrncpy(tableDes->cols[colCount].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - min(TSDB_COL_NAME_LEN + 1, + min(TSDB_COL_NAME_LEN, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes + 1)); tstrncpy(tableDes->cols[colCount].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], @@ -2091,7 +2094,7 @@ static int getTableDes( memset(tableDes->cols[i].value, 0, sizeof(tableDes->cols[i].note)); char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' convertNCharToReadable((char *)row[TSDB_SHOW_TABLES_NAME_INDEX], length[0], tbuf, COL_NOTE_LEN); - sprintf(tableDes->cols[i].value, "\'%s\'", tbuf); + sprintf(tableDes->cols[i].value, "%s", tbuf); break; } case TSDB_DATA_TYPE_TIMESTAMP: diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 553e8446ab449cb3eab8bcc3c15bef8715fe978a..d883daff81d27daa38572fb7fed09add8e56f618 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -145,8 +145,8 @@ static int32_t mnodeCreateCluster() { SClusterObj *pCluster = malloc(sizeof(SClusterObj)); memset(pCluster, 0, sizeof(SClusterObj)); pCluster->createdTime = taosGetTimestampMs(); - bool getuid = taosGetSystemUid(pCluster->uid); - if (!getuid) { + bool mnd_getuid = taosGetSystemUid(pCluster->uid); + if (!mnd_getuid) { strcpy(pCluster->uid, "tdengine2.0"); mError("failed to get uid from system, set to default val %s", pCluster->uid); } else { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 9602e16483f3a367117f998cc619dbe408859d93..daf785cc0770fb68aa020550cc318a71539ca091 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1136,7 +1136,7 @@ static int32_t mnodeAlterDbFp(SMnodeMsg *pMsg) { return sdbUpdateRow(&row); } - //bnNotify(); + bnNotify(); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index bccad221574d49f770bebb61822609c2138f258e..e3051cea81408e58832f7287dda5ec4abba38faf 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -209,7 +209,7 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { mInfos = *pMinfos; } else { mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d", mnodeGetMnodesNum()); - int32_t index = 0; + int32_t mnd_index = 0; void * pIter = NULL; while (1) { SMnodeObj *pMnode = NULL; @@ -219,10 +219,10 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode != NULL) { set = true; - mInfos.mnodeInfos[index].mnodeId = pMnode->mnodeId; - strcpy(mInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp); - if (pMnode->role == TAOS_SYNC_ROLE_MASTER) mInfos.inUse = index; - index++; + mInfos.mnodeInfos[mnd_index].mnodeId = pMnode->mnodeId; + strcpy(mInfos.mnodeInfos[mnd_index].mnodeEp, pDnode->dnodeEp); + if (pMnode->role == TAOS_SYNC_ROLE_MASTER) mInfos.inUse = mnd_index; + mnd_index++; } else { set = false; } @@ -231,7 +231,7 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { mnodeDecMnodeRef(pMnode); } - mInfos.mnodeNum = index; + mInfos.mnodeNum = mnd_index; if (mInfos.mnodeNum < sdbGetReplicaNum()) { set = false; mDebug("vgId:1, mnodes info not synced, current:%d syncCfgNum:%d", mInfos.mnodeNum, sdbGetReplicaNum()); @@ -250,23 +250,23 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { tsMEpForPeer.numOfEps = tsMInfos.mnodeNum; mInfo("vgId:1, mnodes epSet is set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse); - for (int index = 0; index < mInfos.mnodeNum; ++index) { - SMInfo *pInfo = &tsMInfos.mnodeInfos[index]; - taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForShell.fqdn[index], &tsMEpForShell.port[index]); - taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForPeer.fqdn[index], &tsMEpForPeer.port[index]); - tsMEpForPeer.port[index] = tsMEpForPeer.port[index] + TSDB_PORT_DNODEDNODE; + for (int mnd_index = 0; mnd_index < mInfos.mnodeNum; ++mnd_index) { + SMInfo *pInfo = &tsMInfos.mnodeInfos[mnd_index]; + taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForShell.fqdn[mnd_index], &tsMEpForShell.port[mnd_index]); + taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForPeer.fqdn[mnd_index], &tsMEpForPeer.port[mnd_index]); + tsMEpForPeer.port[mnd_index] = tsMEpForPeer.port[mnd_index] + TSDB_PORT_DNODEDNODE; - mInfo("vgId:1, mnode:%d, fqdn:%s shell:%u peer:%u", pInfo->mnodeId, tsMEpForShell.fqdn[index], - tsMEpForShell.port[index], tsMEpForPeer.port[index]); + mInfo("vgId:1, mnode:%d, fqdn:%s shell:%u peer:%u", pInfo->mnodeId, tsMEpForShell.fqdn[mnd_index], + tsMEpForShell.port[mnd_index], tsMEpForPeer.port[mnd_index]); - tsMEpForShell.port[index] = htons(tsMEpForShell.port[index]); - tsMEpForPeer.port[index] = htons(tsMEpForPeer.port[index]); + tsMEpForShell.port[mnd_index] = htons(tsMEpForShell.port[mnd_index]); + tsMEpForPeer.port[mnd_index] = htons(tsMEpForPeer.port[mnd_index]); pInfo->mnodeId = htonl(pInfo->mnodeId); } } else { mInfo("vgId:1, mnodes epSet not set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse); - for (int index = 0; index < tsMInfos.mnodeNum; ++index) { - mInfo("vgId:1, index:%d, ep:%s:%u", index, tsMEpForShell.fqdn[index], htons(tsMEpForShell.port[index])); + for (int mnd_index = 0; mnd_index < tsMInfos.mnodeNum; ++mnd_index) { + mInfo("vgId:1, index:%d, ep:%s:%u", mnd_index, tsMEpForShell.fqdn[mnd_index], htons(tsMEpForShell.port[mnd_index])); } } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 7644f4d7339251037fbe244b17e5af1fa9af0484..661bcc4679dada055a68693dbf3043fd62f05282 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -330,7 +330,7 @@ int32_t sdbUpdateSync(void *pMnodes) { mDebug("vgId:1, update sync config, pMnodes:%p", pMnodes); SSyncCfg syncCfg = {0}; - int32_t index = 0; + int32_t mnd_index = 0; if (pMinfos == NULL) { mDebug("vgId:1, mInfos not input, use mInfos in sdb, numOfMnodes:%d", syncCfg.replica); @@ -341,29 +341,29 @@ int32_t sdbUpdateSync(void *pMnodes) { pIter = mnodeGetNextMnode(pIter, &pMnode); if (pMnode == NULL) break; - syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId; + syncCfg.nodeInfo[mnd_index].nodeId = pMnode->mnodeId; SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode != NULL) { - syncCfg.nodeInfo[index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC; - tstrncpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); - index++; + syncCfg.nodeInfo[mnd_index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC; + tstrncpy(syncCfg.nodeInfo[mnd_index].nodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); + mnd_index++; } mnodeDecDnodeRef(pDnode); mnodeDecMnodeRef(pMnode); } - syncCfg.replica = index; + syncCfg.replica = mnd_index; } else { mDebug("vgId:1, mInfos input, numOfMnodes:%d", pMinfos->mnodeNum); - for (index = 0; index < pMinfos->mnodeNum; ++index) { - SMInfo *node = &pMinfos->mnodeInfos[index]; - syncCfg.nodeInfo[index].nodeId = node->mnodeId; - taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort); - syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC; + for (mnd_index = 0; mnd_index < pMinfos->mnodeNum; ++mnd_index) { + SMInfo *node = &pMinfos->mnodeInfos[mnd_index]; + syncCfg.nodeInfo[mnd_index].nodeId = node->mnodeId; + taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[mnd_index].nodeFqdn, &syncCfg.nodeInfo[mnd_index].nodePort); + syncCfg.nodeInfo[mnd_index].nodePort += TSDB_PORT_SYNC; } - syncCfg.replica = index; + syncCfg.replica = mnd_index; mnodeUpdateMnodeEpSet(pMnodes); } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 570f5c344b624eea1f23fd13f11bfc6e230c61d5..98a41c7a60818a68c8aac0e27abf357b1f82303d 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -121,7 +121,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { } if (!tsMnodeShowMetaFp[pShowMsg->type] || !tsMnodeShowRetrieveFp[pShowMsg->type]) { - mError("show type:%s is not support", mnodeGetShowType(pShowMsg->type)); + mWarn("show type:%s is not support", mnodeGetShowType(pShowMsg->type)); return TSDB_CODE_COM_OPS_NOT_SUPPORT; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index a32da29124016424ce5fe1436f29cca91da3e546..fd6d60c034c702e12a5d996f5b130e54bf3c6a4f 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -298,7 +298,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o if (have) continue; - if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_DROPPING) { + if (/*pVgroup->status == TAOS_VG_STATUS_CREATING ||*/ pVgroup->status == TAOS_VG_STATUS_DROPPING) { mDebug("vgId:%d, not exist in dnode:%d and status is %s, do nothing", pVgroup->vgId, pDnode->dnodeId, vgroupStatus[pVgroup->status]); } else { @@ -742,19 +742,6 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p return 0; } -static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) { - if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) { - return true; - } - - SCTableObj *pCTable = (SCTableObj *)pTable; - if (pVgroup->vgId == pCTable->vgId) { - return true; - } else { - return false; - } -} - static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; SVgObj *pVgroup = NULL; @@ -770,11 +757,6 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v return 0; } - STableObj *pTable = NULL; - if (pShow->payloadLen > 0 ) { - pTable = mnodeGetTable(pShow->payload); - } - while (numOfRows < rows) { pShow->pIter = mnodeGetNextVgroup(pShow->pIter, &pVgroup); if (pVgroup == NULL) break; @@ -784,11 +766,6 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v continue; } - if (!mnodeFilterVgroups(pVgroup, pTable)) { - mnodeDecVgroupRef(pVgroup); - continue; - } - cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -842,7 +819,6 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; - mnodeDecTableRef(pTable); mnodeDecDbRef(pDb); return numOfRows; diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index a64c9d79dd6af511448ad0f9b186f6e50d59c728..bb04a2c6b3d2bbaafef830658eff065ee71b10d1 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -10,5 +10,6 @@ ELSEIF (TD_WINDOWS) ENDIF () ADD_SUBDIRECTORY(src/detail) -ADD_SUBDIRECTORY(tests) - +IF (NOT TD_NINGSI_60) + ADD_SUBDIRECTORY(tests) +ENDIF () diff --git a/src/os/src/darwin/dwSysInfo.c b/src/os/src/darwin/dwSysInfo.c index 10e0acc1309d80154a42b6cc948a8afb7a04b668..f8cd61c7ef80e612ff439b8f696edffc15b3bd56 100644 --- a/src/os/src/darwin/dwSysInfo.c +++ b/src/os/src/darwin/dwSysInfo.c @@ -65,6 +65,8 @@ static void taosGetSystemTimezone() { struct tm tm1; localtime_r(&tx1, &tm1); + tsDaylight = daylight; + /* * format example: * diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index c467c64872d4b660af5cebb19b017f1528b55055..8f06fb1da2fb90a07511c520e89a9ce1f4fdb92a 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -45,8 +45,8 @@ void taosRemoveDir(char *rootDir) { uInfo("dir:%s is removed", rootDir); } -bool taosDirExist(const char* dirname) { - return access(dirname, F_OK) == 0; +bool taosDirExist(const char* os_dirname) { + return access(os_dirname, F_OK) == 0; } int taosMkDir(const char *path, mode_t mode) { diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index cc12968c72eef5b3970ca68cf660de502b402e1e..76ed2196ce7d1e0348d6692dc2b1032926ee6573 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -28,6 +28,7 @@ void taosClose(FileFd fd) { void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; + static uint64_t seqId = 0; int32_t len = (int32_t)strlen(tsTempDir); memcpy(tmpPath, tsTempDir, len); @@ -43,9 +44,11 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, "-%d-%s"); } - char rand[8] = {0}; - taosRandStr(rand, tListLen(rand) - 1); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); + char os_rand[32] = {0}; + + sprintf(os_rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); + + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), os_rand); } #else @@ -68,11 +71,11 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, "-%d-%s"); } - char rand[32] = {0}; + char os_rand[32] = {0}; - sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); + sprintf(os_rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), os_rand); } #endif diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 04b1efe7bf78d57a8806960995b0e34ff79e3abb..8f39459f2ebf95146d6d90c78a4b648e7b2332f3 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -213,6 +213,8 @@ static void taosGetSystemTimezone() { int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; tz += daylight; + tsDaylight = daylight; + /* * format example: * diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index 034ebc28fc51a23893b6cefb9792359c832cd8b0..7b2b157314b612fbd8f0a6f1d4879c322ce6cc4c 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -82,23 +82,23 @@ void deltaToUtcInitOnce() { } static int64_t parseFraction(char* str, char** end, int32_t timePrec); -static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec); -static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec); -static int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec); +static int32_t parseTimeWithTz(char* timestr, int64_t* os_time, int32_t timePrec); +static int32_t parseLocaltime(char* timestr, int64_t* os_time, int32_t timePrec); +static int32_t parseLocaltimeWithDst(char* timestr, int64_t* os_time, int32_t timePrec); -static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = { +static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* os_time, int32_t timePrec) = { parseLocaltime, parseLocaltimeWithDst }; int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { +int32_t taosParseTime(char* timestr, int64_t* os_time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { - return parseTimeWithTz(timestr, time, timePrec); + return parseTimeWithTz(timestr, os_time, timePrec); } else { - return (*parseLocaltimeFp[day_light])(timestr, time, timePrec); + return (*parseLocaltimeFp[day_light])(timestr, os_time, timePrec); } } @@ -213,7 +213,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { * 2013-04-12T15:52:01+0800 * 2013-04-12T15:52:01.123+0800 */ -int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { +int32_t parseTimeWithTz(char* timestr, int64_t* os_time, int32_t timePrec) { int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); @@ -238,14 +238,14 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { if (str[0] == 'Z' || str[0] == 'z') { /* utc time, no millisecond, return directly*/ - *time = seconds * factor; + *os_time = seconds * factor; } else if (str[0] == '.') { str += 1; if ((fraction = parseFraction(str, &str, timePrec)) < 0) { return -1; } - *time = seconds * factor + fraction; + *os_time = seconds * factor + fraction; char seg = str[0]; if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') { @@ -256,18 +256,18 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { return -1; } - *time += tzOffset * factor; + *os_time += tzOffset * factor; } } else if (str[0] == '+' || str[0] == '-') { - *time = seconds * factor + fraction; + *os_time = seconds * factor + fraction; // parse the timezone if (parseTimezone(str, &tzOffset) == -1) { return -1; } - *time += tzOffset * factor; + *os_time += tzOffset * factor; } else { return -1; } @@ -275,8 +275,8 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { return 0; } -int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { - *time = 0; +int32_t parseLocaltime(char* timestr, int64_t* os_time, int32_t timePrec) { + *os_time = 0; struct tm tm = {0}; char* str = strptime(timestr, "%Y-%m-%d %H:%M:%S", &tm); @@ -303,13 +303,13 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); - *time = factor * seconds + fraction; + *os_time = factor * seconds + fraction; return 0; } -int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) { - *time = 0; +int32_t parseLocaltimeWithDst(char* timestr, int64_t* os_time, int32_t timePrec) { + *os_time = 0; struct tm tm = {0}; tm.tm_isdst = -1; @@ -332,73 +332,99 @@ int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) { int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); - *time = factor * seconds + fraction; + *os_time = factor * seconds + fraction; return 0; } -int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { +int64_t convertTimePrecision(int64_t os_time, int32_t fromPrecision, int32_t toPrecision) { assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || toPrecision == TSDB_TIME_PRECISION_NANO); + double tempResult = (double)os_time; switch(fromPrecision) { case TSDB_TIME_PRECISION_MILLI: { switch (toPrecision) { case TSDB_TIME_PRECISION_MILLI: - return time; + return os_time; case TSDB_TIME_PRECISION_MICRO: - return time * 1000; + tempResult *= 1000; + os_time *= 1000; + goto end_; case TSDB_TIME_PRECISION_NANO: - return time * 1000000; + tempResult *= 1000000; + os_time *= 1000000; + goto end_; } } // end from milli case TSDB_TIME_PRECISION_MICRO: { switch (toPrecision) { case TSDB_TIME_PRECISION_MILLI: - return time / 1000; + return os_time / 1000; case TSDB_TIME_PRECISION_MICRO: - return time; + return os_time; case TSDB_TIME_PRECISION_NANO: - return time * 1000; + tempResult *= 1000; + os_time *= 1000; + goto end_; } } //end from micro case TSDB_TIME_PRECISION_NANO: { switch (toPrecision) { case TSDB_TIME_PRECISION_MILLI: - return time / 1000000; + return os_time / 1000000; case TSDB_TIME_PRECISION_MICRO: - return time / 1000; + return os_time / 1000; case TSDB_TIME_PRECISION_NANO: - return time; + return os_time; } } //end from nano default: { assert(0); - return time; // only to pass windows compilation + return os_time; // only to pass windows compilation } } //end switch fromPrecision +end_: + if (tempResult >= (double)INT64_MAX) return INT64_MAX; + if (tempResult <= (double)INT64_MIN) return INT64_MIN + 1; // INT64_MIN means NULL + return os_time; } static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { switch (unit) { - case 's': + case 's':{ + double temp = ((double)val) * MILLISECOND_PER_SECOND; + if (temp >= (double)INT64_MAX || temp <= (double)INT64_MIN) return -1; (*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision); break; - case 'm': + } + case 'm':{ + double temp = ((double)val) * MILLISECOND_PER_MINUTE; + if (temp >= (double)INT64_MAX || temp <= (double)INT64_MIN) return -1; (*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision); break; - case 'h': + } + case 'h':{ + double temp = ((double)val) * MILLISECOND_PER_HOUR; + if (temp >= (double)INT64_MAX || temp <= (double)INT64_MIN) return -1; (*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision); break; - case 'd': + } + case 'd': { + double temp = ((double)val) * MILLISECOND_PER_DAY; + if (temp >= (double)INT64_MAX || temp <= (double)INT64_MIN) return -1; (*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision); break; - case 'w': + } + case 'w': { + double temp = ((double)val) * MILLISECOND_PER_WEEK; + if (temp >= (double)INT64_MAX || temp <= (double)INT64_MIN) return -1; (*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision); break; + } case 'a': (*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_MILLI, timePrecision); break; diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 35ca64d79f8b7a883014fd6ca980300ede22d6e2..84c873202b685e690252890e347632e096a4b39e 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -39,6 +39,20 @@ void osInit() { strcpy(tsDataDir, "/var/lib/ProDB"); strcpy(tsLogDir, "/var/log/ProDB"); strcpy(tsScriptDir, "/etc/ProDB"); +#elif (_TD_KH_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "/etc/kinghistorian"); + } + strcpy(tsDataDir, "/var/lib/kinghistorian"); + strcpy(tsLogDir, "/var/log/kinghistorian"); + strcpy(tsScriptDir, "/etc/kinghistorian"); +#elif (_TD_JH_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "/etc/jh_taos"); + } + strcpy(tsDataDir, "/var/lib/jh_taos"); + strcpy(tsLogDir, "/var/log/jh_taos"); + strcpy(tsScriptDir, "/etc/jh_taos"); #else if (configDir[0] == 0) { strcpy(configDir, "/etc/taos"); diff --git a/src/os/src/linux/osSystem.c b/src/os/src/linux/osSystem.c index a82149dccb1f71e6fbdc2b62d066f04ff52c251d..a5f681d1b2226351f8dad134256e76274de7103c 100644 --- a/src/os/src/linux/osSystem.c +++ b/src/os/src/linux/osSystem.c @@ -33,9 +33,9 @@ void* taosLoadDll(const char *filename) { void* taosLoadSym(void* handle, char* name) { void* sym = dlsym(handle, name); - char* error = NULL; + char* os_error = NULL; - if ((error = dlerror()) != NULL) { + if ((os_error = dlerror()) != NULL) { uWarn("load sym:%s failed, error:%s", name, dlerror()); return NULL; } diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c index 6f46bb43c75ff2c9735fc53a11bce585c1c213f6..6e087c9b29d7468b7c5a4e82c0f69b38f2c01223 100644 --- a/src/os/src/windows/wEnv.c +++ b/src/os/src/windows/wEnv.c @@ -33,12 +33,12 @@ void osInit() { strcpy(tsScriptDir, "C:/PowerDB/script"); #elif (_TD_TQ_ == true) if (configDir[0] == 0) { - strcpy(configDir, "C:/TQ/cfg"); + strcpy(configDir, "C:/TQueue/cfg"); } - strcpy(tsVnodeDir, "C:/TQ/data"); - strcpy(tsDataDir, "C:/TQ/data"); - strcpy(tsLogDir, "C:/TQ/log"); - strcpy(tsScriptDir, "C:/TQ/script"); + strcpy(tsVnodeDir, "C:/TQueue/data"); + strcpy(tsDataDir, "C:/TQueue/data"); + strcpy(tsLogDir, "C:/TQueue/log"); + strcpy(tsScriptDir, "C:/TQueue/script"); #elif (_TD_PRO_ == true) if (configDir[0] == 0) { strcpy(configDir, "C:/ProDB/cfg"); @@ -47,6 +47,22 @@ void osInit() { strcpy(tsDataDir, "C:/ProDB/data"); strcpy(tsLogDir, "C:/ProDB/log"); strcpy(tsScriptDir, "C:/ProDB/script"); +#elif (_TD_KH_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "C:/KingHistorian/cfg"); + } + strcpy(tsVnodeDir, "C:/KingHistorian/data"); + strcpy(tsDataDir, "C:/KingHistorian/data"); + strcpy(tsLogDir, "C:/KingHistorian/log"); + strcpy(tsScriptDir, "C:/KingHistorian/script"); +#elif (_TD_JH_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "C:/jh_iot/cfg"); + } + strcpy(tsVnodeDir, "C:/jh_iot/data"); + strcpy(tsDataDir, "C:/jh_iot/data"); + strcpy(tsLogDir, "C:/jh_iot/log"); + strcpy(tsScriptDir, "C:/jh_iot/script"); #else if (configDir[0] == 0) { strcpy(configDir, "C:/TDengine/cfg"); diff --git a/src/os/src/windows/wString.c b/src/os/src/windows/wString.c index 96a604ae6e3bfb2088a376894c7083a9514ae67f..67afa90121ad64ea5f4387e0ddcc7f0574cfd38f 100644 --- a/src/os/src/windows/wString.c +++ b/src/os/src/windows/wString.c @@ -61,14 +61,14 @@ char *getpass(const char *prefix) { memset(passwd, 0, TSDB_KEY_LEN); printf("%s", prefix); - int32_t index = 0; + int32_t os_index = 0; char ch; - while (index < TSDB_KEY_LEN) { + while (os_index < TSDB_KEY_LEN) { ch = getch(); if (ch == '\n' || ch == '\r') { break; } else { - passwd[index++] = ch; + passwd[os_index++] = ch; } } diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 72793a1049506fed0fce2d1a04c576097fec9fba..6100101207749cc44f0488ca89d0ae04ca0ee7a2 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -91,6 +91,17 @@ static void taosGetSystemTimezone() { strcpy(tsTimezone, tz); } cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char **tzname = _tzname; +#endif +#endif + + tsDaylight = daylight; + uInfo("timezone not configured, use default"); } } diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 6c567e23bc817957d7f376ef101f8e5ca88559e6..af0f4c579557d1bd919847ef858af8dc1616bb6f 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -140,28 +140,29 @@ typedef enum { } EHTTP_CONTEXT_FAILED_CAUSE; typedef struct HttpContext { - int32_t refCount; - SOCKET fd; - uint32_t accessTimes; - uint32_t lastAccessTime; - int32_t state; - uint8_t reqType; - uint8_t parsed; - uint8_t error; - char ipstr[22]; - char user[TSDB_USER_LEN]; // parsed from auth token or login message - char pass[HTTP_PASSWORD_LEN]; - char db[/*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN]; - TAOS * taos; - void * ppContext; - HttpSession *session; - z_stream gzipStream; - HttpParser *parser; - HttpSqlCmd singleCmd; - HttpSqlCmds *multiCmds; - JsonBuf * jsonBuf; - HttpEncodeMethod *encodeMethod; - HttpDecodeMethod *decodeMethod; + int32_t refCount; + SOCKET fd; + uint32_t accessTimes; + uint32_t lastAccessTime; + int32_t state; + uint8_t reqType; + uint8_t parsed; + bool error; + char ipstr[22]; + char user[TSDB_USER_LEN]; // parsed from auth token or login message + char pass[HTTP_PASSWORD_LEN]; + char db[/*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN]; + TAOS * taos; + void * ppContext; + pthread_mutex_t ctxMutex; + HttpSession *session; + z_stream gzipStream; + HttpParser *parser; + HttpSqlCmd singleCmd; + HttpSqlCmds *multiCmds; + JsonBuf *jsonBuf; + HttpEncodeMethod *encodeMethod; + HttpDecodeMethod *decodeMethod; struct HttpThread *pThread; } HttpContext; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 11945453c56ab7fdd1fc8b0c4f2510bbbdda1a6e..4922d98ee492c94e56604d5823e87e4704ddf8e3 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -67,6 +67,8 @@ static void httpDestroyContext(void *data) { pContext->parser = NULL; } + pthread_mutex_destroy(&pContext->ctxMutex); + tfree(pContext); } @@ -118,16 +120,19 @@ HttpContext *httpCreateContext(SOCKET fd) { pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; pContext->parser = httpCreateParser(pContext); + pContext->error = false; TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pContext; HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &pContext, - sizeof(TSDB_CACHE_PTR_TYPE), 3000); + sizeof(TSDB_CACHE_PTR_TYPE), tsHttpKeepAlive); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); // set the ref to 0 taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false); + pthread_mutex_init(&pContext->ctxMutex, NULL); + return pContext; } diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 9719d93824b50064ec1cf23677c641428434592c..6f77994593ebbfc1dc2d9ce97b15a90a797dd8d5 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -45,15 +45,14 @@ bool httpProcessData(HttpContext* pContext) { httpTrace("context:%p, fd:%d, process options request", pContext, pContext->fd); httpSendOptionResp(pContext, "process options request success"); } else { - if (!httpDecodeRequest(pContext)) { - /* - * httpCloseContextByApp has been called when parsing the error - */ - // httpCloseContextByApp(pContext); - } else { + pthread_mutex_lock(&pContext->ctxMutex); + + if (httpDecodeRequest(pContext)) { httpClearParser(pContext->parser); httpProcessRequest(pContext); } + + pthread_mutex_unlock(&pContext->ctxMutex); } return true; diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 1d05b455cb5c66e4f492140e1f337210da04caef..4065f881d1e8a8023e3adacfceaf243de60eebdc 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -52,8 +52,14 @@ static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char * int8_t httpVersion = 0; int8_t keepAlive = 0; + if (pContext->parser != NULL) { httpVersion = pContext->parser->httpVersion; + } + + if (pContext->error == true) { + keepAlive = HTTP_KEEPALIVE_DISABLE; + } else if (pContext->parser != NULL) { keepAlive = pContext->parser->keepAlive; } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 13a0835c3960333c6d12aa443025de5fb95d565e..1b0e60d2ec3575795123967317569e67e007f84a 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -53,7 +53,7 @@ static void httpStopThread(HttpThread *pThread) { break; } } while (0); - if (r) { + if (r && taosCheckPthreadValid(pThread->thread)) { pthread_cancel(pThread->thread); } #else @@ -63,15 +63,21 @@ static void httpStopThread(HttpThread *pThread) { httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); pThread->stop = true; - pthread_cancel(pThread->thread); + if (taosCheckPthreadValid(pThread->thread)) { + pthread_cancel(pThread->thread); + } } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); - pthread_cancel(pThread->thread); + if (taosCheckPthreadValid(pThread->thread)) { + pthread_cancel(pThread->thread); + } } #endif // __APPLE__ - pthread_join(pThread->thread, NULL); + if (taosCheckPthreadValid(pThread->thread)) { + pthread_join(pThread->thread, NULL); + } #ifdef __APPLE__ if (sv[0] != -1) { @@ -398,9 +404,13 @@ static bool httpReadData(HttpContext *pContext) { return true; } } else if (nread < 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + if (errno == EINTR) { + httpDebug("context:%p, fd:%d, read from socket error:%d, continue", pContext, pContext->fd, errno); + continue; + } else if (errno == EAGAIN || errno == EWOULDBLOCK) { httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); - continue; // later again + httpReleaseContext(pContext/*, false */); + return false; } else { httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno); taosCloseSocket(pContext->fd); diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 602767a6563b3ca3430501c0dbcee65333f1d44b..e1b3b17347cba786fa12f5cc2fa7ab3cfb45bd54 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -406,7 +406,14 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { if (pContext->session == NULL) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_SESSION_FULL); } else { + // httpProcessRequestCb called by another thread and a subsequent thread calls this + // function again, if this function called by httpProcessRequestCb executes memset + // just before the subsequent thread executes *Cmd function, nativSql will be NULL + pthread_mutex_lock(&pContext->ctxMutex); + httpExecCmd(pContext); + + pthread_mutex_unlock(&pContext->ctxMutex); } } diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index f30ac7326eef20f4abf5558b288f16f6ee313b42..64893506ffe07f83cd38ca34aa70ede66a38632b 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -31,10 +31,10 @@ bool httpCheckUsedbSql(char *sql) { } bool httpCheckAlterSql(char *sql) { - int32_t index = 0; + int32_t htp_index = 0; do { - SStrToken t0 = tStrGetToken(sql, &index, false); + SStrToken t0 = tStrGetToken(sql, &htp_index, false); if (t0.type != TK_LP) { return t0.type == TK_ALTER; } diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index fea793fa860fd17ff30bcecae1436180bc6b34bf..5677fff71cc56197e9075c286d565d77ce3169c1 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -123,7 +123,11 @@ static void *monThreadFunc(void *param) { if (tsMonitor.quiting) { tsMonitor.state = MON_STATE_NOT_INIT; +#ifdef _TD_KH_ + monInfo("monitor thread will quit, for khserver is quiting"); +#else monInfo("monitor thread will quit, for taosd is quiting"); +#endif break; } else { taosGetDisk(); diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index fd730adee56c3d5edddb943303f5b6b24d9f019c..548bfb3c4a7f2f27700bac55ca325b7469899c13 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -12,10 +12,14 @@ TARGET_LINK_LIBRARIES(query tsdb tutil lua) IF (TD_LINUX) TARGET_LINK_LIBRARIES(query m rt lua) - ADD_SUBDIRECTORY(tests) + IF (NOT TD_NINGSI_60) + ADD_SUBDIRECTORY(tests) + ENDIF () ENDIF () IF (TD_DARWIN) TARGET_LINK_LIBRARIES(query m lua) - ADD_SUBDIRECTORY(tests) + IF (NOT TD_NINGSI_60) + ADD_SUBDIRECTORY(tests) + ENDIF () ENDIF () diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 1f3b6e8465968677d95a3b05d900dc139fc40407..890d7328418c160541522c59136c19f9a35418a1 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -43,6 +43,8 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) +#define RESET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:(((_r)->outputBuf)->info.rows = 0)) + enum { // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, @@ -310,7 +312,8 @@ typedef struct SQueryRuntimeEnv { STableQueryInfo *current; SRspResultInfo resultInfo; SHashObj *pTableRetrieveTsMap; - SUdfInfo *pUdfInfo; + SUdfInfo *pUdfInfo; + bool udfIsCopy; } SQueryRuntimeEnv; enum { @@ -389,6 +392,7 @@ typedef struct SQInfo { int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context int64_t startExecTs; // start to exec timestamp + int64_t lastRetrieveTs; // last retrieve timestamp char* sql; // query sql string SQueryCostInfo summary; } SQInfo; diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index c231c90996e00d84a70c7141eac69c5a59e20254..0ddaabc5fb9bf6eb2c3a16eeedb3b6d952a1f666 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -281,7 +281,7 @@ void *destroyRelationInfo(SRelationInfo* pFromInfo); SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias); // sql expr leaf node -tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType); +tSqlExpr *tSqlExprCreateIdValue(SSqlInfo* pInfo, SStrToken *pToken, int32_t optrType); tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken); diff --git a/src/query/inc/qUdf.h b/src/query/inc/qUdf.h index 1083b1e698f7591aae4586c7722e5343cd9c4d86..77da4b668ae08d10a6154cdece59ec62f5114be9 100644 --- a/src/query/inc/qUdf.h +++ b/src/query/inc/qUdf.h @@ -51,6 +51,7 @@ typedef struct SUdfInfo { SUdfInit init; char *content; char *path; + bool keep; } SUdfInfo; //script diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 920443232e9bfbe59f4b571646793a1c515928bc..cc421b92af333395af9827282eda2edee1dad4f1 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -128,12 +128,6 @@ cmd ::= SHOW dbPrefix(X) VGROUPS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } -cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SStrToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); -} - //drop configure for tables cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { X.n += Z.n; @@ -451,7 +445,17 @@ tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= STRING(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= BOOL(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= NULL(X). { X.type = 0; tVariantCreate(&A, &X); } -tagitem(A) ::= NOW(X). { X.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreate(&A, &X);} +tagitem(A) ::= NOW(X). { X.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreateExt(&A, &X, TK_NOW);} + +tagitem(A) ::= NOW PLUS VARIABLE(X).{ + X.type = TSDB_DATA_TYPE_TIMESTAMP; + tVariantCreateExt(&A, &X, TK_PLUS); +} + +tagitem(A) ::= NOW MINUS VARIABLE(X).{ + X.type = TSDB_DATA_TYPE_TIMESTAMP; + tVariantCreateExt(&A, &X, TK_MINUS); +} tagitem(A) ::= MINUS(X) INTEGER(Y).{ X.n += Y.n; @@ -521,7 +525,7 @@ selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). { } selcollist(A) ::= sclp(P) STAR. { - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); + tSqlExpr *pNode = tSqlExprCreateIdValue(pInfo, NULL, TK_ALL); A = tSqlExprListAppend(P, pNode, 0, 0); } @@ -702,23 +706,23 @@ where_opt(A) ::= WHERE expr(X). {A = X;} expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->exprToken.z = X.z; A->exprToken.n = (Z.z - X.z + 1);} -expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);} -expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= PLUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= MINUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);} -expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);} +expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_ID);} +expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(pInfo, &X, TK_ID);} +expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(pInfo, &X, TK_ALL);} + +expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_INTEGER);} +expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(pInfo, &X, TK_INTEGER);} +expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(pInfo, &X, TK_INTEGER);} +expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_FLOAT);} +expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(pInfo, &X, TK_FLOAT);} +expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(pInfo, &X, TK_FLOAT);} +expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_STRING);} +expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_NOW); } +expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_VARIABLE);} +expr(A) ::= PLUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(pInfo, &X, TK_VARIABLE);} +expr(A) ::= MINUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(pInfo, &X, TK_VARIABLE);} +expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_BOOL);} +expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(pInfo, &X, TK_NULL);} // ordinary functions: min(x), max(x), top(k, 20) expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); } @@ -919,4 +923,4 @@ cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); s %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD LIKE MATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL - NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES. + NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES FILE. diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index da4cb62351b7244c050239688df82bf8bd396afe..6dda83eaba97e3db926963ae9374f9192b370fbf 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -758,7 +758,7 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_ if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { - return (pInfo->ts > w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + return (pInfo->ts >= w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; } } @@ -895,14 +895,14 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } void* tval = NULL; - int16_t index = 0; + int16_t qry_index = 0; if (isMin) { tval = &pCtx->preAggVals.statis.min; - index = pCtx->preAggVals.statis.minIndex; + qry_index = pCtx->preAggVals.statis.minIndex; } else { tval = &pCtx->preAggVals.statis.max; - index = pCtx->preAggVals.statis.maxIndex; + qry_index = pCtx->preAggVals.statis.maxIndex; } TSKEY key = TSKEY_INITIAL_VAL; @@ -913,12 +913,12 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, * * The following codes of 3 lines will be removed later. */ -// if (index < 0 || index >= pCtx->size + pCtx->startOffset) { -// index = 0; +// if (qry_index < 0 || qry_index >= pCtx->size + pCtx->startOffset) { +// qry_index = 0; // } // the index is the original position, not the relative position - key = pCtx->ptsList[index]; + key = pCtx->ptsList[qry_index]; } if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { @@ -1005,11 +1005,11 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; - TSKEY k = tsList[i]; - - DO_UPDATE_TAG_COLUMNS(pCtx, k); + if(tsList) { + TSKEY k = tsList[i]; + DO_UPDATE_TAG_COLUMNS(pCtx, k); + } } - *notNullElems += 1; } #if defined(_DEBUG_VIEW) @@ -1554,15 +1554,15 @@ static void first_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t qry_index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { + if (pInfo->hasResult != DATA_SET_FLAG || timestamp[qry_index] < pInfo->ts) { memcpy(pCtx->pOutput, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; + pInfo->ts = timestamp[qry_index]; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } @@ -1685,19 +1685,19 @@ static void last_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t qry_index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { + if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[qry_index]) { #if defined(_DEBUG_VIEW) qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); #endif memcpy(pCtx->pOutput, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; + pInfo->ts = timestamp[qry_index]; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } @@ -2562,12 +2562,12 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo return true; } -#define LEASTSQR_CAL(p, x, y, index, step) \ +#define LEASTSQR_CAL(p, x, y, qry_index, step) \ do { \ (p)[0][0] += (double)(x) * (x); \ (p)[0][1] += (double)(x); \ - (p)[0][2] += (double)(x) * (y)[index]; \ - (p)[1][2] += (y)[index]; \ + (p)[0][2] += (double)(x) * (y)[qry_index]; \ + (p)[1][2] += (y)[qry_index]; \ (x) += step; \ } while (0) @@ -2992,7 +2992,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { } else { \ *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64)); \ *(type *)(&(ctx)->param[1].i64) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ + *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, qry_index); \ } \ } while (0); @@ -3164,16 +3164,16 @@ static void diff_function(SQLFunctionCtx *pCtx) { char *getArithColumnData(void *param, const char* name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - int32_t index = -1; + int32_t qry_index = -1; for (int32_t i = 0; i < pSupport->numOfCols; ++i) { if (colId == pSupport->colList[i].colId) { - index = i; + qry_index = i; break; } } - assert(index >= 0); - return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; + assert(qry_index >= 0); + return pSupport->data[qry_index] + pSupport->offset * pSupport->colList[qry_index].bytes; } static void arithmetic_function(SQLFunctionCtx *pCtx) { @@ -3381,14 +3381,14 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t qry_index, int32_t size) { int32_t notNullElems = 0; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); TSKEY *tsList = GET_TS_LIST(pCtx); - int32_t i = index; + int32_t i = qry_index; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); SPoint1* last = &pInfo->p; @@ -3399,7 +3399,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si assert(last->key == INT64_MIN); last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, qry_index)); pInfo->dOutput += twa_get_area(pCtx->start, *last); @@ -3409,7 +3409,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si i += step; } else if (pInfo->p.key == INT64_MIN) { last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, qry_index)); pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = last->key; @@ -4246,7 +4246,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " + "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.5g]\n\t " "RowsInMem=[%d] \n\t", percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 101cc3197dfa22499402611c290bbdc3d7123eb3..870dad6f379a336355d00d84373e436105ce01ec 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -138,7 +138,7 @@ do { \ uint64_t queryHandleId = 0; int32_t getMaximumIdleDurationSec() { - return tsShellActivityTimer * 2; + return tsShellActivityTimer * 10; } int64_t genQueryId(void) { int64_t uid = 0; @@ -478,8 +478,8 @@ static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *p existed = (pResultRowInfo->pResult[0] == (*p1)); } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); - int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { + int64_t* qry_index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (qry_index != NULL) { existed = true; } else { existed = false; @@ -517,9 +517,9 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult pResultRowInfo->curPos = 0; } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { - pResultRowInfo->curPos = (int32_t) *index; + int64_t* qry_index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (qry_index != NULL) { + pResultRowInfo->curPos = (int32_t) *qry_index; existed = true; } else { existed = false; @@ -555,9 +555,9 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult pResultRowInfo->curPos = pResultRowInfo->size; pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; - int64_t index = pResultRowInfo->curPos; + int64_t qry_index = pResultRowInfo->curPos; SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &qry_index, POINTER_BYTES); } // too many time window in query @@ -970,13 +970,12 @@ void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); + (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, (int32_t *)&pCtx->resultInfo->numOfRes); } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); + (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, (int32_t *)&pCtx->resultInfo->numOfRes, &pUdfInfo->init); } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { + + if (pCtx->resultInfo->numOfRes > 0) { pCtx->resultInfo->hasResult = DATA_SET_FLAG; } @@ -1167,8 +1166,8 @@ static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY static void setArithParams(SArithmeticSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; sas->pExprInfo = pExprInfo; - if (sas->colList != NULL) { - return; + if (sas->colList) { + free(sas->colList); // free pre malloc memory } sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); for(int32_t i = 0; i < sas->numOfCols; ++i) { @@ -1176,6 +1175,9 @@ static void setArithParams(SArithmeticSupport* sas, SExprInfo *pExprInfo, SSData sas->colList[i] = pColData->info; } + if(sas->data) { + free(sas->data); // free pre malloc memory + } sas->data = calloc(sas->numOfCols, POINTER_BYTES); // set the input column data @@ -1321,14 +1323,14 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } SColIndex * pColIndex = &pExpr[k].base.colInfo; - int16_t index = pColIndex->colIndex; - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); + int16_t qry_index = pColIndex->colIndex; + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, qry_index); assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[qry_index]); } else { GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } @@ -1345,7 +1347,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { if (prevRowIndex == -1) { - pCtx[k].start.ptr = (char *)pRuntimeEnv->prevRow[index]; + pCtx[k].start.ptr = (char *)pRuntimeEnv->prevRow[qry_index]; } else { pCtx[k].start.ptr = (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; } @@ -1711,9 +1713,6 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn int32_t num = 0; for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { char* val = ((char*)pColInfoData->pData) + bytes * j; - if (isNull(val, type)) { - continue; - } // Compare with the previous row of this column, and do not set the output buffer again if they are identical. if (pInfo->prevData == NULL) { @@ -2388,8 +2387,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->sasArray); } - destroyUdfInfo(pRuntimeEnv->pUdfInfo); - + if (!pRuntimeEnv->udfIsCopy) { + destroyUdfInfo(pRuntimeEnv->pUdfInfo); + } + destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); @@ -2429,11 +2430,11 @@ bool isQueryKilled(SQInfo *pQInfo) { // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. - if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs) > getMaximumIdleDurationSec()) && + if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->lastRetrieveTs/1000) > getMaximumIdleDurationSec()) && (!needBuildResAfterQueryComplete(pQInfo))) { assert(pQInfo->startExecTs != 0); - qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d sec, abort current query execution, start:%" PRId64 + qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); return true; } @@ -2774,20 +2775,20 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis if (pDataStatis == NULL || pQueryAttr->numOfFilterCols == 0) { return true; } - bool ret = true; + bool ret = false; for (int32_t k = 0; k < pQueryAttr->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQueryAttr->pFilterInfo[k]; - int32_t index = -1; + int32_t qry_index = -1; for(int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { if (pDataStatis[i].colId == pFilterInfo->info.colId) { - index = i; + qry_index = i; break; } } // no statistics data, load the true data block - if (index == -1) { + if (qry_index == -1) { return true; } @@ -2797,7 +2798,7 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis } // all data in current column are NULL, no need to check its boundary value - if (pDataStatis[index].numOfNull == numOfRows) { + if (pDataStatis[qry_index].numOfNull == numOfRows) { // if isNULL query exists, load the null data column for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { @@ -2810,7 +2811,7 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis continue; } - SDataStatis* pDataBlockst = &pDataStatis[index]; + SDataStatis* pDataBlockst = &pDataStatis[qry_index]; if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) { float minval = (float)(*(double *)(&pDataBlockst->min)); @@ -2818,27 +2819,27 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) { - continue; + return true; //statis filter was not fit for in operator } - ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT); - if (ret == false) { - return false; + ret = pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT); + if (ret) { + return true; //statis filter is coarse, so must return true if anyone is true } } } else { for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) { - continue; + return true; //statis filter was not fit for in operator } - ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type); - if (ret == false) { - return false; + ret = pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type); + if (ret) { + return true; //statis filter is coarse, so must return true if anyone is true } } } } - return ret; + return false; } static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { @@ -4592,10 +4593,10 @@ void queryCostStatis(SQInfo *pQInfo) { // TSKEY key = pTableQueryInfo->win.skey; // // pWindowResInfo->prevSKey = tw.skey; -// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; +// int32_t qry_index = pRuntimeEnv->resultRowInfo.curIndex; // // int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index +// pRuntimeEnv->resultRowInfo.curIndex = qry_index; // restore the window index // // qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, // GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, @@ -5331,15 +5332,15 @@ SArray* getOrderCheckColumns(SQueryAttr* pQuery) { { numOfCols = (int32_t) taosArrayGetSize(pOrderColumns); for(int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); + SColIndex* qry_index = taosArrayGet(pOrderColumns, i); for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { SSqlExpr* pExpr = &pQuery->pExpr1[j].base; int32_t functionId = pExpr->functionId; - if (index->colId == pExpr->colInfo.colId && + if (qry_index->colId == pExpr->colInfo.colId && (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { - index->colIndex = j; - index->colId = pExpr->resColId; + qry_index->colIndex = j; + qry_index->colId = pExpr->resColId; } } } @@ -5359,24 +5360,24 @@ SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { } for(int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); + SColIndex* qry_index = taosArrayGet(pOrderColumns, i); bool found = false; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { SSqlExpr* pExpr = &pQuery->pExpr1[j].base; // TSDB_FUNC_TAG_DUMMY function needs to be ignored - if (index->colId == pExpr->colInfo.colId && + if (qry_index->colId == pExpr->colInfo.colId && ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && ((pExpr->functionId == TSDB_FUNC_TAG) || (pExpr->functionId == TSDB_FUNC_TAGPRJ))) || (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && pExpr->functionId == TSDB_FUNC_PRJ))) { - index->colIndex = j; - index->colId = pExpr->resColId; + qry_index->colIndex = j; + qry_index->colId = pExpr->resColId; found = true; break; } } - assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput); + assert(found && qry_index->colIndex >= 0 && qry_index->colIndex < pQuery->numOfOutput); } return pOrderColumns; @@ -5429,8 +5430,8 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, for(int32_t i = 0; i < numOfCols; ++i) { pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; + SColIndex* qry_index = taosArrayGet(pInfo->orderColumnList, i); + offset += pExpr[qry_index->colIndex].base.resBytes; } numOfCols = (pInfo->groupColumnList != NULL)? (int32_t)taosArrayGetSize(pInfo->groupColumnList):0; @@ -5440,8 +5441,8 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, for(int32_t i = 0; i < numOfCols; ++i) { pInfo->currentGroupColData[i] = (char*)pInfo->currentGroupColData + offset; - SColIndex* index = taosArrayGet(pInfo->groupColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; + SColIndex* qry_index = taosArrayGet(pInfo->groupColumnList, i); + offset += pExpr[qry_index->colIndex].base.resBytes; } initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); @@ -5488,8 +5489,8 @@ SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SEx for(int32_t i = 0; i < numOfCols; ++i) { pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.colBytes; + SColIndex* qry_index = taosArrayGet(pInfo->orderColumnList, i); + offset += pExpr[qry_index->colIndex].base.colBytes; } } @@ -5783,6 +5784,11 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { } } + if (pOperator->status == OP_EXEC_DONE) { + *newgroup = false; + return NULL; + } + while(1) { bool prevVal = *newgroup; @@ -5795,7 +5801,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { //assert(*newgroup == false); *newgroup = prevVal; - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + doSetOperatorCompleted(pOperator); break; } @@ -7050,8 +7056,8 @@ SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator for(int32_t i = 0; i < numOfCols; ++i) { pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; + SColIndex* qry_index = taosArrayGet(pInfo->orderColumnList, i); + offset += pExpr[qry_index->colIndex].base.resBytes; } pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); @@ -7985,7 +7991,7 @@ static char* getUdfFuncName(char* funcname, char* name, int type) { } int32_t initUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { + if (pUdfInfo == NULL || pUdfInfo->handle) { return TSDB_CODE_SUCCESS; } //qError("script len: %d", pUdfInfo->contLen); @@ -8020,10 +8026,21 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { // TODO check for failure of flush to disk /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); fclose(file); - tfree(pUdfInfo->content); + if (!pUdfInfo->keep) { + tfree(pUdfInfo->content); + } + if (pUdfInfo->path) { + unlink(pUdfInfo->path); + } + + tfree(pUdfInfo->path); pUdfInfo->path = strdup(path); + if (pUdfInfo->handle) { + taosCloseDll(pUdfInfo->handle); + } + pUdfInfo->handle = taosLoadDll(path); if (NULL == pUdfInfo->handle) { @@ -8038,9 +8055,17 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); + pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] || NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE]) { + return TSDB_CODE_QRY_SYS_ERROR; + } + } else { + if (pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] || pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE]) { + return TSDB_CODE_QRY_SYS_ERROR; + } } pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); @@ -8147,7 +8172,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp } int32_t param = (int32_t)pExprs[i].base.param[0].i64; - if (pExprs[i].base.functionId != TSDB_FUNC_ARITHM && + if (pExprs[i].base.functionId > 0 && pExprs[i].base.functionId != TSDB_FUNC_ARITHM && (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { tfree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; @@ -8212,11 +8237,11 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t nu type = TSDB_DATA_TYPE_DOUBLE; bytes = tDataTypes[type].bytes; } else { - int32_t index = pExprs[i].base.colInfo.colIndex; - assert(prevExpr[index].base.resColId == pExprs[i].base.colInfo.colId); + int32_t qry_index = pExprs[i].base.colInfo.colIndex; + assert(prevExpr[qry_index].base.resColId == pExprs[i].base.colInfo.colId); - type = prevExpr[index].base.resType; - bytes = prevExpr[index].base.resBytes; + type = prevExpr[qry_index].base.resType; + bytes = prevExpr[qry_index].base.resBytes; } int32_t param = (int32_t)pExprs[i].base.param[0].i64; @@ -8417,6 +8442,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S } pQInfo->qId = qId; + pQInfo->startExecTs = 0; pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; @@ -8559,7 +8585,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; STimeWindow window = pQueryAttr->window; - int32_t index = 0; + int32_t qry_index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); @@ -8575,7 +8601,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S STableKeyInfo* info = taosArrayGet(pa, j); window.skey = info->lastKey; - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); + void* buf = (char*) pQInfo->pBuf + qry_index * sizeof(STableQueryInfo); STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); if (item == NULL) { goto _cleanup; @@ -8586,7 +8612,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S STableId* id = TSDB_TABLEID(info->pTable); taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); - index += 1; + qry_index += 1; } } diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 5152d172649eb91c20ce09c0bf369a5cc3c8de43..87c1d89e1e523bcbfde521a5226f789ffcd465c7 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -448,9 +448,9 @@ int32_t compare_a(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1, int32_t compare_aRv(SSDataBlock* pBlock, SArray* colIndex, int32_t numOfCols, int32_t rowIndex, char** buffer, int32_t order) { for (int32_t i = 0; i < numOfCols; ++i) { SColIndex* pColIndex = taosArrayGet(colIndex, i); - int32_t index = pColIndex->colIndex; + int32_t qry_index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, index); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, qry_index); assert(pColIndex->colId == pColInfo->info.colId); char* data = pColInfo->pData + rowIndex * pColInfo->info.bytes; @@ -1022,14 +1022,14 @@ void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxE } } -SSchema* getColumnModelSchema(SColumnModel *pColumnModel, int32_t index) { - assert(pColumnModel != NULL && index >= 0 && index < pColumnModel->numOfCols); - return &pColumnModel->pFields[index].field; +SSchema* getColumnModelSchema(SColumnModel *pColumnModel, int32_t qry_index) { + assert(pColumnModel != NULL && qry_index >= 0 && qry_index < pColumnModel->numOfCols); + return &pColumnModel->pFields[qry_index].field; } -int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index) { - assert(pColumnModel != NULL && index >= 0 && index < pColumnModel->numOfCols); - return pColumnModel->pFields[index].offset; +int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t qry_index) { + assert(pColumnModel != NULL && qry_index >= 0 && qry_index < pColumnModel->numOfCols); + return pColumnModel->pFields[qry_index].offset; } void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockCapacity, int32_t s, int32_t e) { @@ -1103,17 +1103,17 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) { tfree(pDesc); } -void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) { - assert(numOfRows > 0 && numOfCols > 0 && index >= 0 && index < numOfCols); +void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t qry_index, __compar_fn_t compareFn) { + assert(numOfRows > 0 && numOfCols > 0 && qry_index >= 0 && qry_index < numOfCols); - int32_t bytes = pSchema[index].bytes; + int32_t bytes = pSchema[qry_index].bytes; int32_t size = bytes + sizeof(int32_t); char* buf = calloc(1, size * numOfRows); for(int32_t i = 0; i < numOfRows; ++i) { char* dest = buf + size * i; - memcpy(dest, ((char*) pCols[index]) + bytes * i, bytes); + memcpy(dest, ((char*) pCols[qry_index]) + bytes * i, bytes); *(int32_t*)(dest+bytes) = i; } @@ -1125,7 +1125,7 @@ void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOf for(int32_t i = 0; i < numOfCols; ++i) { int32_t bytes1 = pSchema[i].bytes; - if (i == index) { + if (i == qry_index) { for(int32_t j = 0; j < numOfRows; ++j){ char* src = buf + (j * size); char* dest = ((char*)pCols[i]) + (j * bytes1); diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index 144ca4dd794975a161d85c68e8058e3ca105d9c8..aee2d26b155cf5cdb3f62d6f6aba09318c611df1 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -63,8 +63,8 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); // set the primary timestamp column value - int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); + int32_t qry_index = pFillInfo->numOfCurrent; + char* val = elePtrAt(data[0], TSDB_KEYSIZE, qry_index); *(TSKEY*) val = pFillInfo->currentKey; // set the other values @@ -78,11 +78,11 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData continue; } - char* output = elePtrAt(data[i], pCol->col.bytes, index); + char* output = elePtrAt(data[i], pCol->col.bytes, qry_index); assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); } } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); + setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, qry_index); } } else if (pFillInfo->type == TSDB_FILL_NEXT) { char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; @@ -94,11 +94,11 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData continue; } - char* output = elePtrAt(data[i], pCol->col.bytes, index); + char* output = elePtrAt(data[i], pCol->col.bytes, qry_index); assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); } } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); + setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, qry_index); } } else if (pFillInfo->type == TSDB_FILL_LINEAR) { // TODO : linear interpolation supports NULL value @@ -112,7 +112,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData int16_t type = pCol->col.type; int16_t bytes = pCol->col.bytes; - char *val1 = elePtrAt(data[i], pCol->col.bytes, index); + char *val1 = elePtrAt(data[i], pCol->col.bytes, qry_index); if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { setNull(val1, pCol->col.type, bytes); continue; @@ -124,7 +124,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); } } else { - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); + setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, qry_index); } } else { // fill the default value */ for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { @@ -133,12 +133,12 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData continue; } - char* val1 = elePtrAt(data[i], pCol->col.bytes, index); + char* val1 = elePtrAt(data[i], pCol->col.bytes, qry_index); assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); } } - setTagsValue(pFillInfo, data, index); + setTagsValue(pFillInfo, data, qry_index); pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision); pFillInfo->numOfCurrent++; } @@ -301,11 +301,11 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t numOfTags += 1; bool exists = false; - int32_t index = -1; + int32_t qry_index = -1; for (int32_t j = 0; j < k; ++j) { if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { exists = true; - index = j; + qry_index = j; break; } } @@ -321,7 +321,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t k += 1; } else { - pColInfo->tagIndex = index; + pColInfo->tagIndex = qry_index; } } diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 8544224a647c0497677814ef448498bbf73fab04..aa8acccd5fe5c30b9771b333ca48a037ac9e6cbc 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -45,15 +45,15 @@ //} // ////min heap -// void tHeapAdjust(SHeapEntry* pEntry, int32_t index, int32_t len) { +// void tHeapAdjust(SHeapEntry* pEntry, int32_t qry_index, int32_t len) { // SHeapEntry* ptr = NULL; // // int32_t end = len - 1; // -// SHeapEntry p1 = pEntry[index]; -// int32_t next = index; +// SHeapEntry p1 = pEntry[qry_index]; +// int32_t next = qry_index; // -// for(int32_t i=index; i<=(end-1)/2; ) { +// for(int32_t i=qry_index; i<=(end-1)/2; ) { // int32_t lc = (i<<1) + 1; // int32_t rc = (i+1) << 1; // @@ -119,7 +119,7 @@ // } //} -static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val); +static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t qry_index, double val); SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { /* need one redundant slot */ @@ -390,39 +390,39 @@ static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) { int32_t oldSize = *size; double delta = DBL_MAX; - int32_t index = -1; + int32_t qry_index = -1; for (int32_t i = 1; i < oldSize; ++i) { double d = pHistBin[i].val - pHistBin[i - 1].val; if (d < delta) { delta = d; - index = i - 1; + qry_index = i - 1; } } - SHistBin* s1 = &pHistBin[index]; - SHistBin* s2 = &pHistBin[index + 1]; + SHistBin* s1 = &pHistBin[qry_index]; + SHistBin* s2 = &pHistBin[qry_index + 1]; double newVal = (s1->val * s1->num + s2->val * s2->num) / (s1->num + s2->num); s1->val = newVal; s1->num = s1->num + s2->num; - memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin)); + memmove(&pHistBin[qry_index + 1], &pHistBin[qry_index + 2], (oldSize - qry_index - 2) * sizeof(SHistBin)); (*size) -= 1; #endif } /* optimize this procedure */ -int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { +int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t qry_index, double val) { #if defined(USE_ARRAYLIST) - int32_t remain = pHisto->numOfEntries - index; + int32_t remain = pHisto->numOfEntries - qry_index; if (remain > 0) { - memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); + memmove(&pHisto->elems[qry_index + 1], &pHisto->elems[qry_index], sizeof(SHistBin) * remain); } - assert(index >= 0 && index <= pHisto->maxEntries); + assert(qry_index >= 0 && qry_index <= pHisto->maxEntries); - pHisto->elems[index].num = 1; - pHisto->elems[index].val = val; + pHisto->elems[qry_index].num = 1; + pHisto->elems[qry_index].val = val; pHisto->numOfEntries += 1; /* we need to merge the slot */ diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 024ba77de13086b7ff8e32ab2c4c7340d8806b6b..58275a908976a6f36787d547f0085dc8bd33c4e4 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -122,54 +122,54 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pBucket->type, value); - int32_t index = -1; + int32_t qry_index = -1; if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) { - return index; + return qry_index; } // divide the value range into 1024 buckets uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; if (span < pBucket->numOfSlots) { int64_t delta = v - pBucket->range.i64MinVal; - index = (delta % pBucket->numOfSlots); + qry_index = (delta % pBucket->numOfSlots); } else { double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)(((double)v - pBucket->range.i64MinVal) / slotSpan); - if (index == pBucket->numOfSlots) { - index -= 1; + qry_index = (int32_t)(((double)v - pBucket->range.i64MinVal) / slotSpan); + if (qry_index == pBucket->numOfSlots) { + qry_index -= 1; } } - assert(index >= 0 && index < pBucket->numOfSlots); - return index; + assert(qry_index >= 0 && qry_index < pBucket->numOfSlots); + return qry_index; } int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { int64_t v = 0; GET_TYPED_DATA(v, uint64_t, pBucket->type, value); - int32_t index = -1; + int32_t qry_index = -1; if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) { - return index; + return qry_index; } // divide the value range into 1024 buckets uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal; if (span < pBucket->numOfSlots) { int64_t delta = v - pBucket->range.u64MinVal; - index = (int32_t) (delta % pBucket->numOfSlots); + qry_index = (int32_t) (delta % pBucket->numOfSlots); } else { double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan); + qry_index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan); if (v == pBucket->range.u64MaxVal) { - index -= 1; + qry_index -= 1; } } - assert(index >= 0 && index < pBucket->numOfSlots); - return index; + assert(qry_index >= 0 && qry_index < pBucket->numOfSlots); + return qry_index; } int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { @@ -180,27 +180,27 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { v = GET_DOUBLE_VAL(value); } - int32_t index = -1; + int32_t qry_index = -1; if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { - return index; + return qry_index; } // divide a range of [dMinVal, dMaxVal] into 1024 buckets double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { int32_t delta = (int32_t)(v - pBucket->range.dMinVal); - index = (delta % pBucket->numOfSlots); + qry_index = (delta % pBucket->numOfSlots); } else { double slotSpan = span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); + qry_index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); if (v == pBucket->range.dMaxVal) { - index -= 1; + qry_index -= 1; } } - assert(index >= 0 && index < pBucket->numOfSlots); - return index; + assert(qry_index >= 0 && qry_index < pBucket->numOfSlots); + return qry_index; } static __perc_hash_func_t getHashFunc(int32_t type) { @@ -332,18 +332,18 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { for (int32_t i = 0; i < size; ++i) { char *d = (char *) data + i * bytes; - int32_t index = (pBucket->hashFunc)(pBucket, d); - if (index < 0) { + int32_t qry_index = (pBucket->hashFunc)(pBucket, d); + if (qry_index < 0) { continue; } count += 1; - tMemBucketSlot *pSlot = &pBucket->pSlots[index]; + tMemBucketSlot *pSlot = &pBucket->pSlots[qry_index]; tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); // ensure available memory pages to allocate - int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times); + int32_t groupId = getGroupId(pBucket->numOfSlots, qry_index, pBucket->times); int32_t pageId = -1; if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { @@ -387,7 +387,7 @@ static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int3 return pMemBucket->pSlots[j].range; } -static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); +static bool isIdenticalData(tMemBucket *pMemBucket, int32_t qry_index); static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) { assert(isIdenticalData(pMemBucket, slotIndex)); @@ -532,8 +532,8 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { /* * check if data in one slot are all identical only need to compare with the bounding box */ -bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) { - tMemBucketSlot *pSeg = &pMemBucket->pSlots[index]; +bool isIdenticalData(tMemBucket *pMemBucket, int32_t qry_index) { + tMemBucketSlot *pSeg = &pMemBucket->pSlots[qry_index]; if (IS_FLOAT_TYPE(pMemBucket->type)) { return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON; diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index 1988fc9df7710f15770ca8a9994542d9f4bc8c66..e7d07fc251470ba99d9c1abbb8e18314ffc02319 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -126,9 +126,9 @@ static SQueryNode* doAddTableColumnNode(SQueryInfo* pQueryInfo, STableMetaInfo* for (int32_t i = 0; i < numOfCols; ++i) { SColumn* pCol = taosArrayGetP(tableCols, i); - SColumnIndex index = {.tableIndex = 0, .columnIndex = pCol->columnIndex}; - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SExprInfo* p = tscExprCreate(pTableMetaInfo1, TSDB_FUNC_PRJ, &index, pCol->info.type, pCol->info.bytes, + SColumnIndex qry_index = {.tableIndex = 0, .columnIndex = pCol->columnIndex}; + STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, qry_index.tableIndex); + SExprInfo* p = tscExprCreate(pTableMetaInfo1, TSDB_FUNC_PRJ, &qry_index, pCol->info.type, pCol->info.bytes, pCol->info.colId, 0, TSDB_COL_NORMAL); strncpy(p->base.aliasName, pSchema[pCol->columnIndex].name, tListLen(p->base.aliasName)); @@ -645,6 +645,12 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { } else { op = OP_Project; taosArrayPush(plan, &op); + + if (pQueryAttr->pExpr2 != NULL) { + op = OP_Project; + taosArrayPush(plan, &op); + } + if (pQueryAttr->distinct) { op = OP_Distinct; taosArrayPush(plan, &op); diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index a512c7bf1a2a107998c4555ce9c442e2f501f49c..e465284244fb9a61df3762fd1d2c673ebd87c337 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -134,7 +134,7 @@ SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken) { return pList; } -tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { +tSqlExpr *tSqlExprCreateIdValue(SSqlInfo* pInfo, SStrToken *pToken, int32_t optrType) { tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr)); if (pToken != NULL) { @@ -169,6 +169,7 @@ tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { char unit = 0; int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, &unit, TSDB_TIME_PRECISION_NANO); if (ret != TSDB_CODE_SUCCESS) { + snprintf(pInfo->msg, tListLen(pInfo->msg), "%s", pToken->z); terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } } @@ -538,8 +539,8 @@ SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) { return pList; } -SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) { - if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) { +SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t qry_index) { + if (pList == NULL || pVar == NULL || qry_index >= taosArrayGetSize(pList)) { return tVariantListAppend(NULL, pVar, sortOrder); } @@ -548,7 +549,7 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int item.pVar = *pVar; item.sortOrder = sortOrder; - taosArrayInsert(pList, index, &item); + taosArrayInsert(pList, qry_index, &item); return pList; } @@ -559,7 +560,8 @@ SRelationInfo *setTableNameList(SRelationInfo* pRelationInfo, SStrToken *pName, } pRelationInfo->type = SQL_NODE_FROM_TABLELIST; - SRelElementPair p = {.tableName = *pName}; + SRelElementPair p; + p.tableName = *pName; if (pAlias != NULL) { p.aliasName = *pAlias; } else { @@ -598,7 +600,8 @@ SRelationInfo* addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrT pRelationInfo->type = SQL_NODE_FROM_SUBQUERY; - SRelElementPair p = {.pSubquery = pSub}; + SRelElementPair p; + p.pSubquery = pSub; if (pAlias != NULL) { p.aliasName = *pAlias; } else { @@ -741,6 +744,10 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->bytes = (int16_t)bytes; } + } else { + if (type->type > 0) { + pField->type = -1; + } } } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 99572f6e9345b933434e3685ecb79750a04388fc..9893533a589af0ca7a87dd05628db5059ecbe8eb 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -5,7 +5,7 @@ #include "queryLog.h" static int32_t getDataStartOffset(); -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t qry_index, STSGroupBlockInfo* pBlockInfo); static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); @@ -697,8 +697,8 @@ bool tsBufNextPos(STSBuf* pTSBuf) { int32_t groupIndex = pTSBuf->numOfGroups - 1; pCur->vgroupIndex = groupIndex; - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); + // get current vgroupIndex BlockInfo + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pCur->vgroupIndex].info; int32_t blockIndex = pBlockInfo->numOfBlocks - 1; tsBufGetBlock(pTSBuf, groupIndex, blockIndex); @@ -718,32 +718,43 @@ bool tsBufNextPos(STSBuf* pTSBuf) { while (1) { assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); + // tsIndex is last if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); - if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || + // get current vgroupIndex BlockInfo + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pCur->vgroupIndex].info; + if (pBlockInfo == NULL) { + return false; + } + + // blockIndex is last + if ((pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { + + // vgroupIndex is last if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { + // this is end. both vgroupIndex and blockindex and tsIndex is last pCur->vgroupIndex = -1; return false; } - if (pBlockInfo == NULL) { - return false; - } - + // blockIndex must match with next group + int32_t nextGroupIdx = pCur->vgroupIndex + step; + pBlockInfo = &pTSBuf->pData[nextGroupIdx].info; int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); + // vgroupIndex move next and set value in tsBufGetBlock() tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); break; } else { + // blockIndex move next and set value in tsBufGetBlock() tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step); break; } } else { + // tsIndex move next pCur->tsIndex += step; break; } @@ -767,7 +778,7 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { } STSCursor* pCur = &pTSBuf->cur; - if (pCur != NULL && pCur->vgroupIndex < 0) { + if (pCur->vgroupIndex < 0) { return elem1; } @@ -796,7 +807,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { return -1; } - // src can only have one vnode index + // src can only have one vnode qry_index assert(pSrcBuf->numOfGroups == 1); // there are data in buffer, flush to disk first @@ -819,7 +830,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { pDestBuf->pData = tmp; } - // directly copy the vnode index information + // directly copy the vnode qry_index information memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); // set the new offset value @@ -1012,8 +1023,8 @@ static int32_t getDataStartOffset() { } // update prev vnode length info in file -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) { - int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t qry_index, STSGroupBlockInfo* pBlockInfo) { + int32_t offset = sizeof(STSBufFileHeader) + qry_index * sizeof(STSGroupBlockInfo); doUpdateGroupInfo(pTSBuf, offset, pBlockInfo); } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a26697fb31b5772a2847584a5ea554907058ba4a..57c74c16dfc4761dcdd268468bad1b82e7ed019a 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -168,9 +168,9 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 } // TODO refactor: use macro -SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { - assert(index >= 0 && offset != NULL); - return (SResultRowCellInfo*)((char*) pRow->pCellInfo + offset[index]); +SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t qry_index, int32_t* offset) { + assert(qry_index >= 0 && offset != NULL); + return (SResultRowCellInfo*)((char*) pRow->pCellInfo + offset[qry_index]); } size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 7d30f7c66812c0feb9c7ac37db9d7330fc2f37fb..3538edfcde29ba0cece5fd1e1296b5a4d80c465b 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -35,7 +35,7 @@ typedef struct SQueryMgmt { bool closed; } SQueryMgmt; -static void queryMgmtKillQueryFn(void* handle) { +static void queryMgmtKillQueryFn(void* handle, void* param1) { void** fp = (void**)handle; qKillQuery(*fp); } @@ -213,10 +213,14 @@ bool qTableQuery(qinfo_t qinfo, uint64_t *qId) { } *qId = pQInfo->qId; - pQInfo->startExecTs = taosGetTimestampSec(); + if(pQInfo->startExecTs == 0) { + pQInfo->startExecTs = taosGetTimestampMs(); + pQInfo->lastRetrieveTs = pQInfo->startExecTs; + } if (isQueryKilled(pQInfo)) { qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); + pQInfo->runtimeEnv.outputBuf = NULL; return doBuildResCheck(pQInfo); } @@ -343,6 +347,9 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co setQueryStatus(pRuntimeEnv, QUERY_OVER); } + RESET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)); + pQInfo->lastRetrieveTs = taosGetTimestampMs(); + pQInfo->rspContext = NULL; pQInfo->dataReady = QUERY_RESULT_NOT_READY; @@ -446,7 +453,7 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) { pQueryMgmt->closed = true; pthread_mutex_unlock(&pQueryMgmt->lock); - taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); + taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn, NULL); } void qQueryMgmtReOpen(void *pQMgmt) { @@ -541,3 +548,148 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); return 0; } + +//kill by qid +int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) { + int32_t err = TSDB_CODE_SUCCESS; + void** handle = qAcquireQInfo(pMgmt, qId); + if(handle == NULL) return terrno; + + SQInfo* pQInfo = (SQInfo*)(*handle); + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); + setQueryKilled(pQInfo); + + // wait query stop + int32_t loop = 0; + while (pQInfo->owner != 0) { + taosMsleep(waitMs); + if(loop++ > waitCount){ + err = TSDB_CODE_FAILED; + break; + } + } + + qReleaseQInfo(pMgmt, (void **)&handle, true); + return err; +} + +// local struct +typedef struct { + int64_t qId; + int64_t startExecTs; +} SLongQuery; + +// callbark for sort compare +static int compareLongQuery(const void* p1, const void* p2) { + // sort desc + SLongQuery* plq1 = *(SLongQuery**)p1; + SLongQuery* plq2 = *(SLongQuery**)p2; + if(plq1->startExecTs == plq2->startExecTs) { + return 0; + } else if(plq1->startExecTs > plq2->startExecTs) { + return 1; + } else { + return -1; + } +} + +// callback for taosCacheRefresh +static void cbFoundItem(void* handle, void* param1) { + SQInfo * qInfo = *(SQInfo**) handle; + if(qInfo == NULL) return ; + SArray* qids = (SArray*) param1; + if(qids == NULL) return ; + + bool usedMem = true; + bool usedIMem = true; + SMemTable* mem = qInfo->query.memRef.snapshot.omem; + SMemTable* imem = qInfo->query.memRef.snapshot.imem; + if(mem == NULL || T_REF_VAL_GET(mem) == 0) + usedMem = false; + if(imem == NULL || T_REF_VAL_GET(mem) == 0) + usedIMem = false ; + + if(!usedMem && !usedIMem) + return ; + + // push to qids + SLongQuery* plq = (SLongQuery*)malloc(sizeof(SLongQuery)); + plq->qId = qInfo->qId; + plq->startExecTs = qInfo->startExecTs; + taosArrayPush(qids, &plq); +} + +// longquery +void* qObtainLongQuery(void* param){ + SQueryMgmt* qMgmt = (SQueryMgmt*)param; + if(qMgmt == NULL || qMgmt->qinfoPool == NULL) + return NULL; + SArray* qids = taosArrayInit(4, sizeof(int64_t*)); + if(qids == NULL) return NULL; + // Get each item + taosCacheRefresh(qMgmt->qinfoPool, cbFoundItem, qids); + + size_t cnt = taosArrayGetSize(qids); + if(cnt == 0) { + taosArrayDestroy(qids); + return NULL; + } + if(cnt > 1) + taosArraySort(qids, compareLongQuery); + + return qids; +} + +//solve tsdb no block to commit +bool qFixedNoBlock(void* pRepo, void* pMgmt, int32_t longQueryMs) { + SQueryMgmt *pQueryMgmt = pMgmt; + bool fixed = false; + + // qid top list + SArray *qids = (SArray*)qObtainLongQuery(pQueryMgmt); + if(qids == NULL) return false; + + // kill Query + int64_t now = taosGetTimestampMs(); + size_t cnt = taosArrayGetSize(qids); + size_t i; + SLongQuery* plq; + for(i=0; i < cnt; i++) { + plq = (SLongQuery* )taosArrayGetP(qids, i); + if(plq->startExecTs > now) continue; + if(now - plq->startExecTs >= longQueryMs) { + qKillQueryByQId(pMgmt, plq->qId, 500, 10); // wait 50*100 ms + if(tsdbNoProblem(pRepo)) { + fixed = true; + qWarn("QId:0x%"PRIx64" fixed problem after kill this query.", plq->qId); + break; + } + } + } + + // free qids + for(i=0; i < cnt; i++) { + free(taosArrayGetP(qids, i)); + } + taosArrayDestroy(qids); + return fixed; +} + +//solve tsdb no block to commit +bool qSolveCommitNoBlock(void* pRepo, void* pMgmt) { + qWarn("pRepo=%p start solve problem.", pRepo); + if(qFixedNoBlock(pRepo, pMgmt, 10*60*1000)) { + return true; + } + if(qFixedNoBlock(pRepo, pMgmt, 2*60*1000)){ + return true; + } + if(qFixedNoBlock(pRepo, pMgmt, 30*1000)){ + return true; + } + qWarn("pRepo=%p solve problem failed.", pRepo); + return false; +} \ No newline at end of file diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 15b94f6d4ad0badbcb8b4bea662bac888f8b1047..0b3f561152845e95f3596243113693b7b7812c9f 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -1,3 +1,5 @@ +/* This file is automatically generated by Lemon from input grammar +** source file "sql.y". */ /* ** 2000-05-29 ** @@ -22,9 +24,7 @@ ** The following is the concatenation of all %include directives from the ** input grammar file: */ -#include /************ Begin %include sections from the grammar ************************/ - #include #include #include @@ -37,11 +37,207 @@ #include "tutil.h" #include "tvariant.h" /**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +#ifndef TK_ID +#define TK_ID 1 +#define TK_BOOL 2 +#define TK_TINYINT 3 +#define TK_SMALLINT 4 +#define TK_INTEGER 5 +#define TK_BIGINT 6 +#define TK_FLOAT 7 +#define TK_DOUBLE 8 +#define TK_STRING 9 +#define TK_TIMESTAMP 10 +#define TK_BINARY 11 +#define TK_NCHAR 12 +#define TK_OR 13 +#define TK_AND 14 +#define TK_NOT 15 +#define TK_EQ 16 +#define TK_NE 17 +#define TK_ISNULL 18 +#define TK_NOTNULL 19 +#define TK_IS 20 +#define TK_LIKE 21 +#define TK_GLOB 22 +#define TK_BETWEEN 23 +#define TK_IN 24 +#define TK_GT 25 +#define TK_GE 26 +#define TK_LT 27 +#define TK_LE 28 +#define TK_BITAND 29 +#define TK_BITOR 30 +#define TK_LSHIFT 31 +#define TK_RSHIFT 32 +#define TK_PLUS 33 +#define TK_MINUS 34 +#define TK_DIVIDE 35 +#define TK_TIMES 36 +#define TK_STAR 37 +#define TK_SLASH 38 +#define TK_REM 39 +#define TK_CONCAT 40 +#define TK_UMINUS 41 +#define TK_UPLUS 42 +#define TK_BITNOT 43 +#define TK_SHOW 44 +#define TK_DATABASES 45 +#define TK_TOPICS 46 +#define TK_FUNCTIONS 47 +#define TK_MNODES 48 +#define TK_DNODES 49 +#define TK_ACCOUNTS 50 +#define TK_USERS 51 +#define TK_MODULES 52 +#define TK_QUERIES 53 +#define TK_CONNECTIONS 54 +#define TK_STREAMS 55 +#define TK_VARIABLES 56 +#define TK_SCORES 57 +#define TK_GRANTS 58 +#define TK_VNODES 59 +#define TK_IPTOKEN 60 +#define TK_DOT 61 +#define TK_CREATE 62 +#define TK_TABLE 63 +#define TK_STABLE 64 +#define TK_DATABASE 65 +#define TK_TABLES 66 +#define TK_STABLES 67 +#define TK_VGROUPS 68 +#define TK_DROP 69 +#define TK_TOPIC 70 +#define TK_FUNCTION 71 +#define TK_DNODE 72 +#define TK_USER 73 +#define TK_ACCOUNT 74 +#define TK_USE 75 +#define TK_DESCRIBE 76 +#define TK_DESC 77 +#define TK_ALTER 78 +#define TK_PASS 79 +#define TK_PRIVILEGE 80 +#define TK_LOCAL 81 +#define TK_COMPACT 82 +#define TK_LP 83 +#define TK_RP 84 +#define TK_IF 85 +#define TK_EXISTS 86 +#define TK_AS 87 +#define TK_OUTPUTTYPE 88 +#define TK_AGGREGATE 89 +#define TK_BUFSIZE 90 +#define TK_PPS 91 +#define TK_TSERIES 92 +#define TK_DBS 93 +#define TK_STORAGE 94 +#define TK_QTIME 95 +#define TK_CONNS 96 +#define TK_STATE 97 +#define TK_COMMA 98 +#define TK_KEEP 99 +#define TK_CACHE 100 +#define TK_REPLICA 101 +#define TK_QUORUM 102 +#define TK_DAYS 103 +#define TK_MINROWS 104 +#define TK_MAXROWS 105 +#define TK_BLOCKS 106 +#define TK_CTIME 107 +#define TK_WAL 108 +#define TK_FSYNC 109 +#define TK_COMP 110 +#define TK_PRECISION 111 +#define TK_UPDATE 112 +#define TK_CACHELAST 113 +#define TK_PARTITIONS 114 +#define TK_UNSIGNED 115 +#define TK_TAGS 116 +#define TK_USING 117 +#define TK_NULL 118 +#define TK_NOW 119 +#define TK_VARIABLE 120 +#define TK_SELECT 121 +#define TK_UNION 122 +#define TK_ALL 123 +#define TK_DISTINCT 124 +#define TK_FROM 125 +#define TK_INTERVAL 126 +#define TK_EVERY 127 +#define TK_SESSION 128 +#define TK_STATE_WINDOW 129 +#define TK_FILL 130 +#define TK_SLIDING 131 +#define TK_ORDER 132 +#define TK_BY 133 +#define TK_ASC 134 +#define TK_GROUP 135 +#define TK_HAVING 136 +#define TK_LIMIT 137 +#define TK_OFFSET 138 +#define TK_SLIMIT 139 +#define TK_SOFFSET 140 +#define TK_WHERE 141 +#define TK_RESET 142 +#define TK_QUERY 143 +#define TK_SYNCDB 144 +#define TK_ADD 145 +#define TK_COLUMN 146 +#define TK_MODIFY 147 +#define TK_TAG 148 +#define TK_CHANGE 149 +#define TK_SET 150 +#define TK_KILL 151 +#define TK_CONNECTION 152 +#define TK_STREAM 153 +#define TK_COLON 154 +#define TK_ABORT 155 +#define TK_AFTER 156 +#define TK_ATTACH 157 +#define TK_BEFORE 158 +#define TK_BEGIN 159 +#define TK_CASCADE 160 +#define TK_CLUSTER 161 +#define TK_CONFLICT 162 +#define TK_COPY 163 +#define TK_DEFERRED 164 +#define TK_DELIMITERS 165 +#define TK_DETACH 166 +#define TK_EACH 167 +#define TK_END 168 +#define TK_EXPLAIN 169 +#define TK_FAIL 170 +#define TK_FOR 171 +#define TK_IGNORE 172 +#define TK_IMMEDIATE 173 +#define TK_INITIALLY 174 +#define TK_INSTEAD 175 +#define TK_MATCH 176 +#define TK_KEY 177 +#define TK_OF 178 +#define TK_RAISE 179 +#define TK_REPLACE 180 +#define TK_RESTRICT 181 +#define TK_ROW 182 +#define TK_STATEMENT 183 +#define TK_TRIGGER 184 +#define TK_VIEW 185 +#define TK_SEMI 186 +#define TK_NONE 187 +#define TK_PREV 188 +#define TK_LINEAR 189 +#define TK_IMPORT 190 +#define TK_TBNAME 191 +#define TK_JOIN 192 +#define TK_INSERT 193 +#define TK_INTO 194 +#define TK_VALUES 195 +#define TK_FILE 196 +#endif +/**************** End token definitions ***************************************/ /* The next sections is a series of control #defines. ** various aspects of the generated parser. @@ -76,8 +272,10 @@ ** zero the stack is dynamically sized using realloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser +** ParseCTX_* As ParseARG_ except for %extra_context ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -97,7 +295,7 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 279 +#define YYNOCODE 278 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { @@ -126,21 +324,29 @@ typedef union { #endif #define ParseARG_SDECL SSqlInfo* pInfo; #define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo -#define ParseARG_STORE yypParser->pInfo = pInfo +#define ParseARG_PARAM ,pInfo +#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; +#define ParseARG_STORE yypParser->pInfo=pInfo; +#define ParseCTX_SDECL +#define ParseCTX_PDECL +#define ParseCTX_PARAM +#define ParseCTX_FETCH +#define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 364 -#define YYNRULE 292 -#define YYNTOKEN 196 -#define YY_MAX_SHIFT 363 -#define YY_MIN_SHIFTREDUCE 572 -#define YY_MAX_SHIFTREDUCE 863 -#define YY_ERROR_ACTION 864 -#define YY_ACCEPT_ACTION 865 -#define YY_NO_ACTION 866 -#define YY_MIN_REDUCE 867 -#define YY_MAX_REDUCE 1158 +#define YYNSTATE 366 +#define YYNRULE 293 +#define YYNRULE_WITH_ACTION 293 +#define YYNTOKEN 197 +#define YY_MAX_SHIFT 365 +#define YY_MIN_SHIFTREDUCE 575 +#define YY_MAX_SHIFTREDUCE 867 +#define YY_ERROR_ACTION 868 +#define YY_ACCEPT_ACTION 869 +#define YY_NO_ACTION 870 +#define YY_MIN_REDUCE 871 +#define YY_MAX_REDUCE 1163 /************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -205,84 +411,85 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (758) +#define YY_ACTTAB_COUNT (761) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 170, 624, 236, 624, 230, 361, 1023, 1045, 242, 625, - /* 10 */ 247, 625, 1023, 57, 58, 154, 61, 62, 281, 38, - /* 20 */ 250, 51, 624, 60, 319, 65, 63, 66, 64, 993, - /* 30 */ 625, 991, 992, 56, 55, 161, 994, 54, 53, 52, - /* 40 */ 995, 161, 996, 997, 865, 363, 1036, 573, 574, 575, - /* 50 */ 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, - /* 60 */ 586, 362, 233, 232, 231, 57, 58, 1020, 61, 62, - /* 70 */ 208, 660, 250, 51, 1042, 60, 319, 65, 63, 66, - /* 80 */ 64, 1135, 1009, 279, 278, 56, 55, 80, 98, 54, - /* 90 */ 53, 52, 57, 58, 1036, 61, 62, 208, 86, 250, - /* 100 */ 51, 1017, 60, 319, 65, 63, 66, 64, 1134, 1084, - /* 110 */ 272, 291, 56, 55, 317, 1083, 54, 53, 52, 57, - /* 120 */ 59, 244, 61, 62, 1011, 1023, 250, 51, 95, 60, - /* 130 */ 319, 65, 63, 66, 64, 45, 802, 349, 1036, 56, - /* 140 */ 55, 161, 1022, 54, 53, 52, 58, 243, 61, 62, - /* 150 */ 767, 768, 250, 51, 234, 60, 319, 65, 63, 66, - /* 160 */ 64, 1006, 1007, 35, 1010, 56, 55, 317, 246, 54, - /* 170 */ 53, 52, 44, 315, 356, 355, 314, 313, 312, 354, - /* 180 */ 311, 310, 309, 353, 308, 352, 351, 985, 973, 974, - /* 190 */ 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, - /* 200 */ 986, 987, 61, 62, 24, 23, 250, 51, 264, 60, - /* 210 */ 319, 65, 63, 66, 64, 92, 79, 268, 267, 56, - /* 220 */ 55, 123, 211, 54, 53, 52, 249, 817, 208, 217, - /* 230 */ 806, 93, 809, 349, 812, 137, 136, 135, 216, 1135, - /* 240 */ 249, 817, 324, 86, 806, 81, 809, 259, 812, 251, - /* 250 */ 1008, 205, 65, 63, 66, 64, 161, 175, 228, 229, - /* 260 */ 56, 55, 320, 916, 54, 53, 52, 5, 41, 179, - /* 270 */ 189, 624, 228, 229, 178, 104, 109, 100, 108, 625, - /* 280 */ 45, 732, 357, 954, 729, 38, 730, 38, 731, 121, - /* 290 */ 115, 126, 253, 304, 38, 16, 125, 15, 131, 134, - /* 300 */ 124, 206, 271, 38, 78, 258, 784, 128, 67, 208, - /* 310 */ 748, 224, 255, 256, 3, 190, 199, 197, 195, 808, - /* 320 */ 1135, 811, 67, 194, 141, 140, 139, 138, 293, 240, - /* 330 */ 91, 241, 926, 1020, 44, 1020, 356, 355, 328, 189, - /* 340 */ 38, 354, 1020, 818, 813, 353, 38, 352, 351, 708, - /* 350 */ 814, 1019, 807, 29, 810, 212, 38, 818, 813, 38, - /* 360 */ 1094, 56, 55, 783, 814, 54, 53, 52, 254, 38, - /* 370 */ 252, 38, 327, 326, 38, 14, 1131, 339, 338, 94, - /* 380 */ 917, 260, 745, 257, 329, 334, 333, 189, 1020, 259, - /* 390 */ 330, 54, 53, 52, 1020, 259, 752, 733, 734, 176, - /* 400 */ 331, 1, 177, 335, 1020, 1021, 273, 1020, 9, 97, - /* 410 */ 83, 84, 34, 336, 71, 337, 764, 1020, 341, 1020, - /* 420 */ 39, 774, 1020, 360, 359, 146, 152, 150, 149, 775, - /* 430 */ 156, 74, 718, 804, 321, 68, 296, 720, 77, 815, - /* 440 */ 248, 133, 132, 26, 298, 299, 39, 275, 275, 719, - /* 450 */ 39, 68, 838, 819, 623, 114, 72, 113, 96, 18, - /* 460 */ 1130, 17, 6, 68, 1129, 226, 25, 25, 25, 805, - /* 470 */ 707, 75, 816, 737, 735, 738, 736, 20, 227, 19, - /* 480 */ 120, 22, 119, 21, 209, 210, 213, 207, 214, 1154, - /* 490 */ 215, 1146, 1037, 1093, 219, 238, 220, 221, 218, 204, - /* 500 */ 1090, 1089, 239, 340, 269, 153, 1044, 1055, 48, 1076, - /* 510 */ 1052, 1053, 276, 1075, 1018, 1057, 151, 155, 160, 287, - /* 520 */ 171, 172, 280, 1016, 173, 174, 163, 235, 282, 284, - /* 530 */ 931, 164, 165, 1034, 166, 763, 301, 821, 302, 303, - /* 540 */ 306, 307, 46, 202, 162, 294, 42, 290, 168, 318, - /* 550 */ 925, 325, 76, 73, 50, 292, 1153, 288, 111, 1152, - /* 560 */ 283, 1149, 286, 49, 180, 305, 332, 1145, 167, 122, - /* 570 */ 350, 117, 1144, 1141, 181, 951, 43, 40, 47, 342, - /* 580 */ 203, 913, 127, 911, 129, 130, 909, 908, 261, 192, - /* 590 */ 193, 905, 904, 903, 902, 901, 900, 899, 196, 198, - /* 600 */ 896, 894, 892, 890, 200, 887, 201, 343, 274, 82, - /* 610 */ 87, 344, 285, 1077, 345, 346, 347, 348, 358, 863, - /* 620 */ 263, 262, 862, 266, 225, 245, 300, 265, 861, 844, - /* 630 */ 843, 270, 222, 275, 10, 85, 295, 930, 929, 223, - /* 640 */ 105, 740, 106, 277, 30, 88, 765, 907, 906, 157, - /* 650 */ 158, 142, 143, 898, 184, 183, 952, 187, 182, 185, - /* 660 */ 186, 188, 144, 897, 776, 145, 953, 889, 989, 2, - /* 670 */ 169, 888, 770, 33, 4, 159, 89, 237, 772, 90, - /* 680 */ 289, 999, 31, 11, 32, 12, 13, 27, 297, 28, - /* 690 */ 97, 99, 102, 36, 101, 638, 37, 103, 673, 671, - /* 700 */ 670, 669, 667, 666, 665, 662, 628, 316, 107, 7, - /* 710 */ 822, 820, 322, 8, 323, 710, 110, 69, 112, 70, - /* 720 */ 39, 709, 116, 118, 706, 654, 652, 644, 650, 646, - /* 730 */ 648, 642, 640, 676, 675, 674, 672, 668, 664, 663, - /* 740 */ 191, 590, 626, 588, 867, 866, 866, 866, 866, 866, - /* 750 */ 866, 866, 866, 866, 866, 866, 147, 148, + /* 0 */ 169, 626, 235, 626, 229, 363, 1026, 1050, 241, 627, + /* 10 */ 246, 627, 1026, 57, 58, 153, 61, 62, 281, 38, + /* 20 */ 249, 51, 626, 60, 319, 65, 63, 66, 64, 996, + /* 30 */ 627, 994, 995, 56, 55, 160, 997, 54, 53, 52, + /* 40 */ 998, 160, 999, 1000, 869, 365, 1041, 576, 577, 578, + /* 50 */ 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, + /* 60 */ 589, 364, 232, 231, 230, 57, 58, 1023, 61, 62, + /* 70 */ 207, 662, 249, 51, 1047, 60, 319, 65, 63, 66, + /* 80 */ 64, 1140, 23, 279, 278, 56, 55, 80, 98, 54, + /* 90 */ 53, 52, 57, 58, 1041, 61, 62, 207, 1041, 249, + /* 100 */ 51, 1020, 60, 319, 65, 63, 66, 64, 1139, 1089, + /* 110 */ 272, 291, 56, 55, 233, 1088, 54, 53, 52, 57, + /* 120 */ 59, 243, 61, 62, 1014, 1026, 249, 51, 95, 60, + /* 130 */ 319, 65, 63, 66, 64, 812, 806, 815, 317, 56, + /* 140 */ 55, 160, 123, 54, 53, 52, 58, 242, 61, 62, + /* 150 */ 771, 772, 249, 51, 351, 60, 319, 65, 63, 66, + /* 160 */ 64, 1009, 1010, 35, 1013, 56, 55, 86, 245, 54, + /* 170 */ 53, 52, 44, 315, 358, 357, 314, 313, 312, 356, + /* 180 */ 311, 310, 309, 355, 308, 354, 353, 988, 976, 977, + /* 190 */ 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, + /* 200 */ 989, 990, 61, 62, 24, 45, 249, 51, 264, 60, + /* 210 */ 319, 65, 63, 66, 64, 92, 1012, 268, 267, 56, + /* 220 */ 55, 204, 210, 54, 53, 52, 248, 821, 207, 216, + /* 230 */ 810, 741, 813, 742, 816, 137, 136, 135, 215, 1140, + /* 240 */ 248, 821, 324, 86, 810, 811, 813, 814, 816, 250, + /* 250 */ 819, 205, 65, 63, 66, 64, 919, 160, 227, 228, + /* 260 */ 56, 55, 320, 188, 54, 53, 52, 5, 41, 178, + /* 270 */ 259, 626, 227, 228, 177, 104, 109, 100, 108, 627, + /* 280 */ 174, 45, 259, 749, 734, 259, 38, 731, 38, 732, + /* 290 */ 252, 733, 175, 304, 351, 1024, 121, 115, 126, 258, + /* 300 */ 211, 317, 271, 125, 78, 131, 134, 124, 67, 207, + /* 310 */ 9, 223, 359, 957, 128, 254, 255, 198, 196, 194, + /* 320 */ 1140, 38, 67, 1025, 193, 141, 140, 139, 138, 293, + /* 330 */ 239, 91, 240, 38, 1023, 44, 1023, 358, 357, 34, + /* 340 */ 38, 38, 356, 822, 817, 818, 355, 38, 354, 353, + /* 350 */ 56, 55, 752, 29, 54, 53, 52, 822, 817, 818, + /* 360 */ 820, 38, 38, 38, 38, 328, 253, 788, 251, 1023, + /* 370 */ 327, 326, 299, 93, 710, 260, 273, 257, 321, 336, + /* 380 */ 335, 1022, 14, 79, 329, 330, 94, 81, 1023, 1023, + /* 390 */ 39, 331, 54, 53, 52, 1023, 362, 361, 603, 1159, + /* 400 */ 735, 256, 341, 340, 83, 337, 338, 339, 343, 1023, + /* 410 */ 1023, 1023, 1023, 151, 149, 148, 97, 1011, 929, 920, + /* 420 */ 1, 176, 3, 189, 787, 188, 188, 84, 808, 768, + /* 430 */ 778, 779, 71, 74, 247, 720, 296, 722, 298, 756, + /* 440 */ 721, 842, 275, 155, 68, 26, 823, 1136, 625, 39, + /* 450 */ 39, 68, 96, 77, 68, 25, 6, 16, 1135, 15, + /* 460 */ 25, 739, 25, 740, 809, 275, 1134, 114, 18, 113, + /* 470 */ 17, 333, 332, 75, 72, 20, 225, 19, 120, 269, + /* 480 */ 119, 825, 22, 226, 21, 709, 133, 132, 208, 209, + /* 490 */ 212, 1151, 1099, 152, 206, 213, 1098, 214, 218, 237, + /* 500 */ 219, 220, 1095, 217, 203, 1094, 238, 342, 1049, 48, + /* 510 */ 1060, 1042, 1057, 1058, 1062, 150, 276, 274, 154, 159, + /* 520 */ 1081, 1080, 170, 287, 1021, 171, 1019, 172, 173, 767, + /* 530 */ 166, 934, 280, 301, 1039, 234, 162, 302, 303, 306, + /* 540 */ 307, 46, 201, 282, 42, 284, 161, 294, 318, 928, + /* 550 */ 76, 73, 50, 163, 292, 164, 290, 165, 325, 1158, + /* 560 */ 111, 1157, 288, 286, 167, 1154, 179, 334, 1150, 49, + /* 570 */ 283, 117, 1149, 305, 1146, 180, 954, 43, 40, 47, + /* 580 */ 202, 916, 127, 914, 129, 130, 912, 911, 261, 191, + /* 590 */ 192, 908, 907, 906, 905, 904, 903, 902, 195, 197, + /* 600 */ 898, 896, 894, 199, 891, 200, 352, 82, 122, 87, + /* 610 */ 344, 285, 1082, 345, 346, 347, 348, 349, 350, 224, + /* 620 */ 360, 244, 300, 867, 262, 263, 866, 265, 266, 865, + /* 630 */ 848, 221, 105, 933, 932, 222, 106, 847, 270, 275, + /* 640 */ 295, 10, 85, 744, 277, 88, 30, 910, 909, 901, + /* 650 */ 182, 955, 185, 142, 183, 181, 900, 187, 184, 186, + /* 660 */ 143, 144, 992, 145, 893, 892, 769, 956, 168, 33, + /* 670 */ 2, 4, 156, 158, 780, 157, 236, 774, 89, 90, + /* 680 */ 1002, 776, 289, 31, 11, 32, 12, 13, 27, 297, + /* 690 */ 28, 97, 99, 102, 36, 101, 640, 37, 103, 675, + /* 700 */ 673, 672, 671, 669, 668, 667, 664, 316, 107, 7, + /* 710 */ 8, 630, 824, 322, 323, 110, 826, 112, 69, 70, + /* 720 */ 738, 737, 712, 711, 116, 118, 708, 656, 39, 654, + /* 730 */ 646, 652, 648, 650, 644, 642, 678, 677, 676, 674, + /* 740 */ 670, 666, 665, 190, 628, 593, 591, 871, 870, 870, + /* 750 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 146, + /* 760 */ 147, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 253, 1, 245, 1, 199, 200, 249, 200, 245, 9, @@ -293,75 +500,75 @@ static const YYCODETYPE yy_lookahead[] = { /* 50 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, /* 60 */ 58, 59, 263, 244, 62, 13, 14, 248, 16, 17, /* 70 */ 266, 5, 20, 21, 267, 23, 24, 25, 26, 27, - /* 80 */ 28, 277, 0, 268, 269, 33, 34, 87, 207, 37, - /* 90 */ 38, 39, 13, 14, 247, 16, 17, 266, 83, 20, + /* 80 */ 28, 277, 266, 268, 269, 33, 34, 87, 207, 37, + /* 90 */ 38, 39, 13, 14, 247, 16, 17, 266, 247, 20, /* 100 */ 21, 200, 23, 24, 25, 26, 27, 28, 277, 274, - /* 110 */ 263, 276, 33, 34, 85, 274, 37, 38, 39, 13, + /* 110 */ 263, 276, 33, 34, 263, 274, 37, 38, 39, 13, /* 120 */ 14, 245, 16, 17, 243, 249, 20, 21, 207, 23, - /* 130 */ 24, 25, 26, 27, 28, 120, 84, 91, 247, 33, - /* 140 */ 34, 200, 249, 37, 38, 39, 14, 246, 16, 17, - /* 150 */ 126, 127, 20, 21, 263, 23, 24, 25, 26, 27, - /* 160 */ 28, 240, 241, 242, 243, 33, 34, 85, 206, 37, + /* 130 */ 24, 25, 26, 27, 28, 5, 84, 7, 85, 33, + /* 140 */ 34, 200, 79, 37, 38, 39, 14, 246, 16, 17, + /* 150 */ 126, 127, 20, 21, 91, 23, 24, 25, 26, 27, + /* 160 */ 28, 240, 241, 242, 243, 33, 34, 83, 206, 37, /* 170 */ 38, 39, 99, 100, 101, 102, 103, 104, 105, 106, /* 180 */ 107, 108, 109, 110, 111, 112, 113, 223, 224, 225, /* 190 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 200 */ 236, 237, 16, 17, 44, 266, 20, 21, 143, 23, - /* 210 */ 24, 25, 26, 27, 28, 274, 207, 152, 153, 33, - /* 220 */ 34, 79, 62, 37, 38, 39, 1, 2, 266, 69, - /* 230 */ 5, 250, 7, 91, 9, 75, 76, 77, 78, 277, - /* 240 */ 1, 2, 82, 83, 5, 264, 7, 200, 9, 206, - /* 250 */ 241, 266, 25, 26, 27, 28, 200, 210, 33, 34, - /* 260 */ 33, 34, 37, 205, 37, 38, 39, 63, 64, 65, - /* 270 */ 212, 1, 33, 34, 70, 71, 72, 73, 74, 9, - /* 280 */ 120, 2, 221, 222, 5, 200, 7, 200, 9, 63, - /* 290 */ 64, 65, 69, 89, 200, 146, 70, 148, 72, 73, - /* 300 */ 74, 266, 142, 200, 144, 69, 77, 81, 83, 266, - /* 310 */ 37, 151, 33, 34, 203, 204, 63, 64, 65, 5, - /* 320 */ 277, 7, 83, 70, 71, 72, 73, 74, 272, 244, - /* 330 */ 274, 244, 205, 248, 99, 248, 101, 102, 244, 212, - /* 340 */ 200, 106, 248, 118, 119, 110, 200, 112, 113, 5, - /* 350 */ 125, 248, 5, 83, 7, 266, 200, 118, 119, 200, - /* 360 */ 239, 33, 34, 134, 125, 37, 38, 39, 145, 200, - /* 370 */ 147, 200, 149, 150, 200, 83, 266, 33, 34, 87, - /* 380 */ 205, 145, 98, 147, 244, 149, 150, 212, 248, 200, - /* 390 */ 244, 37, 38, 39, 248, 200, 123, 118, 119, 210, - /* 400 */ 244, 208, 209, 244, 248, 210, 84, 248, 124, 117, - /* 410 */ 84, 84, 83, 244, 98, 244, 84, 248, 244, 248, - /* 420 */ 98, 84, 248, 66, 67, 68, 63, 64, 65, 84, - /* 430 */ 98, 98, 84, 1, 15, 98, 84, 84, 83, 125, - /* 440 */ 61, 79, 80, 98, 84, 116, 98, 121, 121, 84, - /* 450 */ 98, 98, 84, 84, 84, 146, 140, 148, 98, 146, - /* 460 */ 266, 148, 83, 98, 266, 266, 98, 98, 98, 37, - /* 470 */ 115, 138, 125, 5, 5, 7, 7, 146, 266, 148, - /* 480 */ 146, 146, 148, 148, 266, 266, 266, 266, 266, 249, - /* 490 */ 266, 249, 247, 239, 266, 239, 266, 266, 266, 266, - /* 500 */ 239, 239, 239, 239, 200, 200, 200, 200, 265, 275, - /* 510 */ 200, 200, 247, 275, 247, 200, 61, 200, 200, 200, - /* 520 */ 251, 200, 270, 200, 200, 200, 260, 270, 270, 270, - /* 530 */ 200, 259, 258, 262, 257, 125, 200, 118, 200, 200, - /* 540 */ 200, 200, 200, 200, 261, 132, 200, 130, 255, 200, - /* 550 */ 200, 200, 137, 139, 136, 135, 200, 129, 200, 200, - /* 560 */ 131, 200, 128, 141, 200, 90, 200, 200, 256, 97, - /* 570 */ 114, 200, 200, 200, 200, 200, 200, 200, 200, 96, + /* 200 */ 236, 237, 16, 17, 44, 121, 20, 21, 143, 23, + /* 210 */ 24, 25, 26, 27, 28, 274, 0, 152, 153, 33, + /* 220 */ 34, 266, 62, 37, 38, 39, 1, 2, 266, 69, + /* 230 */ 5, 5, 7, 7, 9, 75, 76, 77, 78, 277, + /* 240 */ 1, 2, 82, 83, 5, 5, 7, 7, 9, 206, + /* 250 */ 120, 266, 25, 26, 27, 28, 205, 200, 33, 34, + /* 260 */ 33, 34, 37, 212, 37, 38, 39, 63, 64, 65, + /* 270 */ 200, 1, 33, 34, 70, 71, 72, 73, 74, 9, + /* 280 */ 210, 121, 200, 98, 2, 200, 200, 5, 200, 7, + /* 290 */ 69, 9, 210, 89, 91, 210, 63, 64, 65, 69, + /* 300 */ 266, 85, 142, 70, 144, 72, 73, 74, 83, 266, + /* 310 */ 125, 151, 221, 222, 81, 33, 34, 63, 64, 65, + /* 320 */ 277, 200, 83, 249, 70, 71, 72, 73, 74, 272, + /* 330 */ 244, 274, 244, 200, 248, 99, 248, 101, 102, 83, + /* 340 */ 200, 200, 106, 118, 119, 120, 110, 200, 112, 113, + /* 350 */ 33, 34, 37, 83, 37, 38, 39, 118, 119, 120, + /* 360 */ 120, 200, 200, 200, 200, 244, 145, 77, 147, 248, + /* 370 */ 149, 150, 116, 250, 5, 145, 84, 147, 15, 149, + /* 380 */ 150, 248, 83, 207, 244, 244, 87, 264, 248, 248, + /* 390 */ 98, 244, 37, 38, 39, 248, 66, 67, 68, 249, + /* 400 */ 118, 119, 33, 34, 84, 244, 244, 244, 244, 248, + /* 410 */ 248, 248, 248, 63, 64, 65, 117, 241, 205, 205, + /* 420 */ 208, 209, 203, 204, 134, 212, 212, 84, 1, 84, + /* 430 */ 84, 84, 98, 98, 61, 84, 84, 84, 84, 124, + /* 440 */ 84, 84, 122, 98, 98, 98, 84, 266, 84, 98, + /* 450 */ 98, 98, 98, 83, 98, 98, 83, 146, 266, 148, + /* 460 */ 98, 5, 98, 7, 37, 122, 266, 146, 146, 148, + /* 470 */ 148, 33, 34, 138, 140, 146, 266, 148, 146, 200, + /* 480 */ 148, 118, 146, 266, 148, 115, 79, 80, 266, 266, + /* 490 */ 266, 249, 239, 200, 266, 266, 239, 266, 266, 239, + /* 500 */ 266, 266, 239, 266, 266, 239, 239, 239, 200, 265, + /* 510 */ 200, 247, 200, 200, 200, 61, 247, 201, 200, 200, + /* 520 */ 275, 275, 251, 200, 247, 200, 200, 200, 200, 120, + /* 530 */ 256, 200, 270, 200, 262, 270, 260, 200, 200, 200, + /* 540 */ 200, 200, 200, 270, 200, 270, 261, 132, 200, 200, + /* 550 */ 137, 139, 136, 259, 135, 258, 130, 257, 200, 200, + /* 560 */ 200, 200, 129, 128, 255, 200, 200, 200, 200, 141, + /* 570 */ 131, 200, 200, 90, 200, 200, 200, 200, 200, 200, /* 580 */ 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, /* 590 */ 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, - /* 600 */ 200, 200, 200, 200, 200, 200, 200, 51, 201, 201, - /* 610 */ 201, 93, 201, 201, 95, 55, 94, 92, 85, 5, - /* 620 */ 5, 154, 5, 5, 201, 201, 201, 154, 5, 101, - /* 630 */ 100, 143, 201, 121, 83, 122, 116, 211, 211, 201, - /* 640 */ 207, 84, 207, 98, 83, 98, 84, 201, 201, 83, - /* 650 */ 83, 202, 202, 201, 214, 218, 220, 216, 219, 217, - /* 660 */ 215, 213, 202, 201, 84, 202, 222, 201, 238, 208, - /* 670 */ 254, 201, 84, 252, 203, 98, 83, 1, 84, 83, - /* 680 */ 83, 238, 98, 133, 98, 133, 83, 83, 116, 83, - /* 690 */ 117, 79, 71, 88, 87, 5, 88, 87, 9, 5, - /* 700 */ 5, 5, 5, 5, 5, 5, 86, 15, 79, 83, - /* 710 */ 118, 84, 24, 83, 59, 5, 148, 16, 148, 16, - /* 720 */ 98, 5, 148, 148, 84, 5, 5, 5, 5, 5, + /* 600 */ 200, 200, 200, 200, 200, 200, 114, 201, 97, 201, + /* 610 */ 96, 201, 201, 51, 93, 95, 55, 94, 92, 201, + /* 620 */ 85, 201, 201, 5, 154, 5, 5, 154, 5, 5, + /* 630 */ 101, 201, 207, 211, 211, 201, 207, 100, 143, 122, + /* 640 */ 116, 83, 123, 84, 98, 98, 83, 201, 201, 201, + /* 650 */ 218, 220, 215, 202, 214, 219, 201, 213, 217, 216, + /* 660 */ 202, 202, 238, 202, 201, 201, 84, 222, 254, 252, + /* 670 */ 208, 203, 83, 98, 84, 83, 1, 84, 83, 83, + /* 680 */ 238, 84, 83, 98, 133, 98, 133, 83, 83, 116, + /* 690 */ 83, 117, 79, 71, 88, 87, 5, 88, 87, 9, + /* 700 */ 5, 5, 5, 5, 5, 5, 5, 15, 79, 83, + /* 710 */ 83, 86, 84, 24, 59, 148, 118, 148, 16, 16, + /* 720 */ 120, 120, 5, 5, 148, 148, 84, 5, 98, 5, /* 730 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 740 */ 98, 61, 86, 60, 0, 278, 278, 278, 278, 278, - /* 750 */ 278, 278, 278, 278, 278, 278, 21, 21, 278, 278, - /* 760 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + /* 740 */ 5, 5, 5, 98, 86, 61, 60, 0, 278, 278, + /* 750 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 21, + /* 760 */ 21, 278, 278, 278, 278, 278, 278, 278, 278, 278, /* 770 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, /* 780 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, /* 790 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, @@ -380,114 +587,114 @@ static const YYCODETYPE yy_lookahead[] = { /* 920 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, /* 930 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, /* 940 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 950 */ 278, 278, 278, 278, + /* 950 */ 278, 278, 278, 278, 278, 278, 278, 278, }; -#define YY_SHIFT_COUNT (363) +#define YY_SHIFT_COUNT (365) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (744) +#define YY_SHIFT_MAX (747) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 160, 73, 73, 235, 235, 29, 225, 239, 239, 270, + /* 0 */ 160, 73, 73, 236, 236, 53, 225, 239, 239, 270, /* 10 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - /* 20 */ 21, 21, 21, 0, 2, 239, 279, 279, 279, 15, - /* 30 */ 15, 21, 21, 24, 21, 82, 21, 21, 21, 21, - /* 40 */ 142, 29, 46, 46, 66, 758, 758, 758, 239, 239, + /* 20 */ 21, 21, 21, 0, 2, 239, 282, 282, 282, 84, + /* 30 */ 84, 21, 21, 24, 21, 216, 21, 21, 21, 21, + /* 40 */ 63, 53, 203, 203, 66, 761, 761, 761, 239, 239, /* 50 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, - /* 60 */ 239, 239, 239, 239, 239, 239, 239, 239, 279, 279, - /* 70 */ 279, 344, 344, 344, 344, 344, 344, 344, 21, 21, - /* 80 */ 21, 273, 21, 21, 21, 15, 15, 21, 21, 21, - /* 90 */ 21, 229, 229, 284, 15, 21, 21, 21, 21, 21, + /* 60 */ 239, 239, 239, 239, 239, 239, 239, 239, 282, 282, + /* 70 */ 282, 369, 369, 369, 369, 369, 369, 369, 21, 21, + /* 80 */ 21, 315, 21, 21, 21, 84, 84, 21, 21, 21, + /* 90 */ 21, 290, 290, 185, 84, 21, 21, 21, 21, 21, /* 100 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, /* 110 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, /* 120 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, /* 130 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, /* 140 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - /* 150 */ 21, 21, 21, 455, 455, 455, 410, 410, 410, 410, - /* 160 */ 455, 455, 415, 414, 413, 418, 420, 417, 428, 434, - /* 170 */ 429, 422, 455, 455, 455, 475, 475, 456, 29, 29, - /* 180 */ 455, 455, 472, 483, 556, 518, 519, 560, 522, 525, - /* 190 */ 456, 66, 455, 455, 533, 533, 455, 533, 455, 533, - /* 200 */ 455, 455, 758, 758, 52, 79, 79, 106, 79, 132, - /* 210 */ 186, 204, 227, 227, 227, 227, 226, 253, 328, 328, - /* 220 */ 328, 328, 223, 236, 65, 292, 354, 354, 314, 347, - /* 230 */ 357, 363, 322, 326, 327, 332, 337, 345, 316, 333, - /* 240 */ 348, 352, 353, 360, 365, 329, 368, 369, 432, 379, - /* 250 */ 419, 370, 149, 309, 313, 468, 469, 331, 334, 355, - /* 260 */ 335, 362, 614, 467, 615, 617, 473, 618, 623, 528, - /* 270 */ 530, 488, 512, 520, 551, 513, 557, 561, 545, 547, - /* 280 */ 562, 566, 580, 567, 588, 577, 593, 594, 596, 676, - /* 290 */ 597, 584, 550, 586, 552, 603, 520, 604, 572, 606, - /* 300 */ 573, 612, 605, 607, 621, 690, 608, 610, 689, 694, - /* 310 */ 695, 696, 697, 698, 699, 700, 620, 692, 629, 626, - /* 320 */ 627, 592, 630, 688, 655, 701, 568, 570, 622, 622, - /* 330 */ 622, 622, 703, 574, 575, 622, 622, 622, 710, 716, - /* 340 */ 640, 622, 720, 721, 722, 723, 724, 725, 726, 727, - /* 350 */ 728, 729, 730, 731, 732, 733, 734, 642, 656, 735, - /* 360 */ 736, 680, 683, 744, + /* 150 */ 21, 21, 454, 454, 454, 409, 409, 409, 409, 454, + /* 160 */ 454, 413, 412, 415, 416, 419, 426, 433, 435, 439, + /* 170 */ 428, 454, 454, 454, 483, 483, 492, 53, 53, 454, + /* 180 */ 454, 511, 514, 562, 521, 520, 561, 523, 526, 492, + /* 190 */ 66, 454, 454, 535, 535, 454, 535, 454, 535, 454, + /* 200 */ 454, 761, 761, 52, 79, 79, 106, 79, 132, 186, + /* 210 */ 204, 227, 227, 227, 227, 233, 254, 317, 317, 317, + /* 220 */ 317, 221, 230, 65, 299, 355, 355, 130, 240, 330, + /* 230 */ 350, 292, 320, 343, 345, 346, 347, 334, 335, 351, + /* 240 */ 352, 353, 354, 356, 256, 357, 362, 427, 373, 363, + /* 250 */ 364, 311, 321, 322, 226, 456, 438, 329, 332, 370, + /* 260 */ 336, 407, 618, 470, 620, 621, 473, 623, 624, 529, + /* 270 */ 537, 495, 517, 524, 558, 519, 559, 563, 546, 547, + /* 280 */ 582, 589, 590, 592, 593, 575, 595, 597, 596, 675, + /* 290 */ 599, 585, 551, 587, 553, 604, 524, 605, 573, 607, + /* 300 */ 574, 613, 606, 608, 622, 691, 609, 611, 690, 695, + /* 310 */ 696, 697, 698, 699, 700, 701, 625, 692, 629, 626, + /* 320 */ 628, 598, 627, 689, 655, 702, 567, 569, 630, 630, + /* 330 */ 630, 630, 600, 601, 703, 576, 577, 630, 630, 630, + /* 340 */ 717, 718, 642, 630, 722, 724, 725, 726, 727, 728, + /* 350 */ 729, 730, 731, 732, 733, 734, 735, 736, 737, 645, + /* 360 */ 658, 738, 739, 684, 686, 747, }; -#define YY_REDUCE_COUNT (203) +#define YY_REDUCE_COUNT (202) #define YY_REDUCE_MIN (-253) -#define YY_REDUCE_MAX (471) +#define YY_REDUCE_MAX (468) static const short yy_reduce_ofst[] = { /* 0 */ -153, -36, -36, -194, -194, -79, -196, -38, 43, -185, - /* 10 */ -181, -165, 56, 85, 87, 94, 140, 146, 156, 159, - /* 20 */ 169, 171, 174, -193, -195, -169, -243, -237, -124, -201, - /* 30 */ -109, -159, -59, -253, -99, -119, 47, 189, 195, 103, - /* 40 */ 58, 9, 127, 175, 61, -19, 193, 111, -61, -15, - /* 50 */ 35, 89, 110, 194, 198, 199, 212, 218, 219, 220, - /* 60 */ 221, 222, 224, 228, 230, 231, 232, 233, -107, 240, - /* 70 */ 242, 121, 254, 256, 261, 262, 263, 264, 304, 305, - /* 80 */ 306, 243, 307, 310, 311, 245, 265, 315, 317, 318, - /* 90 */ 319, 234, 238, 269, 267, 321, 323, 324, 325, 330, - /* 100 */ 336, 338, 339, 340, 341, 342, 343, 346, 349, 350, - /* 110 */ 351, 356, 358, 359, 361, 364, 366, 367, 371, 372, - /* 120 */ 373, 374, 375, 376, 377, 378, 380, 381, 382, 383, + /* 10 */ -181, -165, 57, 86, 88, 121, 140, 141, 147, 161, + /* 20 */ 162, 163, 164, -193, -195, -169, -243, -237, -124, -201, + /* 30 */ -149, -159, -59, -253, -99, -119, 70, 82, 85, 133, + /* 40 */ 51, 176, 213, 214, 91, 123, 212, 219, -184, -45, + /* 50 */ -15, 34, 181, 192, 200, 210, 217, 222, 223, 224, + /* 60 */ 228, 229, 231, 232, 234, 235, 237, 238, 74, 150, + /* 70 */ 242, 253, 257, 260, 263, 266, 267, 268, 279, 293, + /* 80 */ 308, 244, 310, 312, 313, 264, 269, 314, 318, 319, + /* 90 */ 323, 245, 246, 271, 277, 325, 326, 327, 328, 331, + /* 100 */ 333, 337, 338, 339, 340, 341, 342, 344, 348, 349, + /* 110 */ 358, 359, 360, 361, 365, 366, 367, 368, 371, 372, + /* 120 */ 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, /* 130 */ 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, /* 140 */ 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, - /* 150 */ 404, 405, 406, 407, 408, 409, 252, 257, 258, 259, - /* 160 */ 411, 412, 271, 283, 266, 272, 274, 277, 312, 293, - /* 170 */ 416, 421, 423, 424, 425, 426, 427, 430, 433, 435, - /* 180 */ 431, 438, 436, 439, 437, 440, 442, 445, 441, 448, - /* 190 */ 443, 444, 446, 447, 449, 450, 452, 460, 462, 463, - /* 200 */ 466, 470, 461, 471, + /* 150 */ 404, 405, 316, 406, 408, 262, 265, 273, 275, 410, + /* 160 */ 411, 272, 285, 276, 294, 297, 300, 274, 309, 414, + /* 170 */ 417, 418, 420, 421, 422, 423, 424, 425, 429, 430, + /* 180 */ 434, 431, 436, 432, 440, 441, 437, 443, 444, 442, + /* 190 */ 445, 446, 447, 451, 458, 448, 459, 455, 461, 463, + /* 200 */ 464, 462, 468, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 864, 988, 927, 998, 914, 924, 1137, 1137, 1137, 864, - /* 10 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 20 */ 864, 864, 864, 1046, 884, 1137, 864, 864, 864, 864, - /* 30 */ 864, 864, 864, 1061, 864, 924, 864, 864, 864, 864, - /* 40 */ 934, 924, 934, 934, 864, 1041, 972, 990, 864, 864, - /* 50 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 60 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 70 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 80 */ 864, 1048, 1054, 1051, 864, 864, 864, 1056, 864, 864, - /* 90 */ 864, 1080, 1080, 1039, 864, 864, 864, 864, 864, 864, - /* 100 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 110 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 120 */ 864, 864, 864, 864, 864, 864, 864, 912, 864, 910, - /* 130 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 140 */ 864, 864, 864, 864, 864, 864, 895, 864, 864, 864, - /* 150 */ 864, 864, 864, 886, 886, 886, 864, 864, 864, 864, - /* 160 */ 886, 886, 1087, 1091, 1073, 1085, 1081, 1068, 1066, 1064, - /* 170 */ 1072, 1095, 886, 886, 886, 932, 932, 928, 924, 924, - /* 180 */ 886, 886, 950, 948, 946, 938, 944, 940, 942, 936, - /* 190 */ 915, 864, 886, 886, 922, 922, 886, 922, 886, 922, - /* 200 */ 886, 886, 972, 990, 864, 1096, 1086, 864, 1136, 1126, - /* 210 */ 1125, 864, 1132, 1124, 1123, 1122, 864, 864, 1118, 1121, - /* 220 */ 1120, 1119, 864, 864, 864, 864, 1128, 1127, 864, 864, - /* 230 */ 864, 864, 864, 864, 864, 864, 864, 864, 1092, 1088, - /* 240 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 1098, - /* 250 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 1000, - /* 260 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 270 */ 864, 864, 1038, 864, 864, 864, 864, 864, 1050, 1049, - /* 280 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 290 */ 864, 1082, 864, 1074, 864, 864, 1012, 864, 864, 864, - /* 300 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 310 */ 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, - /* 320 */ 864, 864, 864, 864, 864, 864, 864, 864, 1155, 1150, - /* 330 */ 1151, 1148, 864, 864, 864, 1147, 1142, 1143, 864, 864, - /* 340 */ 864, 1140, 864, 864, 864, 864, 864, 864, 864, 864, - /* 350 */ 864, 864, 864, 864, 864, 864, 864, 956, 864, 893, - /* 360 */ 891, 864, 882, 864, + /* 0 */ 868, 991, 930, 1001, 917, 927, 1142, 1142, 1142, 868, + /* 10 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 20 */ 868, 868, 868, 1051, 888, 1142, 868, 868, 868, 868, + /* 30 */ 868, 868, 868, 1066, 868, 927, 868, 868, 868, 868, + /* 40 */ 937, 927, 937, 937, 868, 1046, 975, 993, 868, 868, + /* 50 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 60 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 70 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 80 */ 868, 1053, 1059, 1056, 868, 868, 868, 1061, 868, 868, + /* 90 */ 868, 1085, 1085, 1044, 868, 868, 868, 868, 868, 868, + /* 100 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 110 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 120 */ 868, 868, 868, 868, 868, 868, 868, 915, 868, 913, + /* 130 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 140 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 150 */ 868, 868, 890, 890, 890, 868, 868, 868, 868, 890, + /* 160 */ 890, 1092, 1096, 1078, 1090, 1086, 1073, 1071, 1069, 1077, + /* 170 */ 1100, 890, 890, 890, 935, 935, 931, 927, 927, 890, + /* 180 */ 890, 953, 951, 949, 941, 947, 943, 945, 939, 918, + /* 190 */ 868, 890, 890, 925, 925, 890, 925, 890, 925, 890, + /* 200 */ 890, 975, 993, 868, 1101, 1091, 868, 1141, 1131, 1130, + /* 210 */ 868, 1137, 1129, 1128, 1127, 868, 868, 1123, 1126, 1125, + /* 220 */ 1124, 868, 868, 868, 868, 1133, 1132, 868, 868, 868, + /* 230 */ 868, 868, 868, 868, 868, 868, 868, 1097, 1093, 868, + /* 240 */ 868, 868, 868, 868, 868, 868, 868, 868, 1103, 868, + /* 250 */ 868, 868, 868, 868, 868, 868, 1032, 868, 868, 1003, + /* 260 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 270 */ 868, 868, 1043, 868, 868, 868, 868, 868, 1055, 1054, + /* 280 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 290 */ 868, 1087, 868, 1079, 868, 868, 1015, 868, 868, 868, + /* 300 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 310 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 868, + /* 320 */ 868, 868, 868, 868, 868, 868, 868, 868, 1160, 1155, + /* 330 */ 1156, 1153, 868, 868, 868, 868, 868, 1152, 1147, 1148, + /* 340 */ 868, 868, 868, 1145, 868, 868, 868, 868, 868, 868, + /* 350 */ 868, 868, 868, 868, 868, 868, 868, 868, 868, 959, + /* 360 */ 868, 897, 895, 868, 886, 868, }; /********** End of lemon-generated parsing tables *****************************/ @@ -627,12 +834,12 @@ static const YYCODETYPE yyFallback[] = { 0, /* USING => nothing */ 1, /* NULL => ID */ 1, /* NOW => ID */ + 0, /* VARIABLE => nothing */ 0, /* SELECT => nothing */ 0, /* UNION => nothing */ 1, /* ALL => ID */ 0, /* DISTINCT => nothing */ 0, /* FROM => nothing */ - 0, /* VARIABLE => nothing */ 0, /* INTERVAL => nothing */ 0, /* EVERY => nothing */ 0, /* SESSION => nothing */ @@ -703,6 +910,7 @@ static const YYCODETYPE yyFallback[] = { 1, /* INSERT => ID */ 1, /* INTO => ID */ 1, /* VALUES => ID */ + 1, /* FILE => ID */ }; #endif /* YYFALLBACK */ @@ -742,6 +950,7 @@ struct yyParser { int yyerrcnt; /* Shifts left before out of the error */ #endif ParseARG_SDECL /* A place to hold %extra_argument */ + ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ @@ -755,6 +964,7 @@ typedef struct yyParser yyParser; #ifndef NDEBUG #include +#include static FILE *yyTraceFILE = 0; static char *yyTracePrompt = 0; #endif /* NDEBUG */ @@ -909,12 +1119,12 @@ static const char *const yyTokenName[] = { /* 117 */ "USING", /* 118 */ "NULL", /* 119 */ "NOW", - /* 120 */ "SELECT", - /* 121 */ "UNION", - /* 122 */ "ALL", - /* 123 */ "DISTINCT", - /* 124 */ "FROM", - /* 125 */ "VARIABLE", + /* 120 */ "VARIABLE", + /* 121 */ "SELECT", + /* 122 */ "UNION", + /* 123 */ "ALL", + /* 124 */ "DISTINCT", + /* 125 */ "FROM", /* 126 */ "INTERVAL", /* 127 */ "EVERY", /* 128 */ "SESSION", @@ -985,7 +1195,7 @@ static const char *const yyTokenName[] = { /* 193 */ "INSERT", /* 194 */ "INTO", /* 195 */ "VALUES", - /* 196 */ "error", + /* 196 */ "FILE", /* 197 */ "program", /* 198 */ "cmd", /* 199 */ "dbPrefix", @@ -1103,269 +1313,270 @@ static const char *const yyRuleName[] = { /* 26 */ "cmd ::= SHOW dbPrefix STABLES", /* 27 */ "cmd ::= SHOW dbPrefix STABLES LIKE ids", /* 28 */ "cmd ::= SHOW dbPrefix VGROUPS", - /* 29 */ "cmd ::= SHOW dbPrefix VGROUPS ids", - /* 30 */ "cmd ::= DROP TABLE ifexists ids cpxName", - /* 31 */ "cmd ::= DROP STABLE ifexists ids cpxName", - /* 32 */ "cmd ::= DROP DATABASE ifexists ids", - /* 33 */ "cmd ::= DROP TOPIC ifexists ids", - /* 34 */ "cmd ::= DROP FUNCTION ids", - /* 35 */ "cmd ::= DROP DNODE ids", - /* 36 */ "cmd ::= DROP USER ids", - /* 37 */ "cmd ::= DROP ACCOUNT ids", - /* 38 */ "cmd ::= USE ids", - /* 39 */ "cmd ::= DESCRIBE ids cpxName", - /* 40 */ "cmd ::= DESC ids cpxName", - /* 41 */ "cmd ::= ALTER USER ids PASS ids", - /* 42 */ "cmd ::= ALTER USER ids PRIVILEGE ids", - /* 43 */ "cmd ::= ALTER DNODE ids ids", - /* 44 */ "cmd ::= ALTER DNODE ids ids ids", - /* 45 */ "cmd ::= ALTER LOCAL ids", - /* 46 */ "cmd ::= ALTER LOCAL ids ids", - /* 47 */ "cmd ::= ALTER DATABASE ids alter_db_optr", - /* 48 */ "cmd ::= ALTER TOPIC ids alter_topic_optr", - /* 49 */ "cmd ::= ALTER ACCOUNT ids acct_optr", - /* 50 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", - /* 51 */ "cmd ::= COMPACT VNODES IN LP exprlist RP", - /* 52 */ "ids ::= ID", - /* 53 */ "ids ::= STRING", - /* 54 */ "ifexists ::= IF EXISTS", - /* 55 */ "ifexists ::=", - /* 56 */ "ifnotexists ::= IF NOT EXISTS", - /* 57 */ "ifnotexists ::=", - /* 58 */ "cmd ::= CREATE DNODE ids", - /* 59 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", - /* 60 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", - /* 61 */ "cmd ::= CREATE TOPIC ifnotexists ids topic_optr", - /* 62 */ "cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", - /* 63 */ "cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", - /* 64 */ "cmd ::= CREATE USER ids PASS ids", - /* 65 */ "bufsize ::=", - /* 66 */ "bufsize ::= BUFSIZE INTEGER", - /* 67 */ "pps ::=", - /* 68 */ "pps ::= PPS INTEGER", - /* 69 */ "tseries ::=", - /* 70 */ "tseries ::= TSERIES INTEGER", - /* 71 */ "dbs ::=", - /* 72 */ "dbs ::= DBS INTEGER", - /* 73 */ "streams ::=", - /* 74 */ "streams ::= STREAMS INTEGER", - /* 75 */ "storage ::=", - /* 76 */ "storage ::= STORAGE INTEGER", - /* 77 */ "qtime ::=", - /* 78 */ "qtime ::= QTIME INTEGER", - /* 79 */ "users ::=", - /* 80 */ "users ::= USERS INTEGER", - /* 81 */ "conns ::=", - /* 82 */ "conns ::= CONNS INTEGER", - /* 83 */ "state ::=", - /* 84 */ "state ::= STATE ids", - /* 85 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", - /* 86 */ "intitemlist ::= intitemlist COMMA intitem", - /* 87 */ "intitemlist ::= intitem", - /* 88 */ "intitem ::= INTEGER", - /* 89 */ "keep ::= KEEP intitemlist", - /* 90 */ "cache ::= CACHE INTEGER", - /* 91 */ "replica ::= REPLICA INTEGER", - /* 92 */ "quorum ::= QUORUM INTEGER", - /* 93 */ "days ::= DAYS INTEGER", - /* 94 */ "minrows ::= MINROWS INTEGER", - /* 95 */ "maxrows ::= MAXROWS INTEGER", - /* 96 */ "blocks ::= BLOCKS INTEGER", - /* 97 */ "ctime ::= CTIME INTEGER", - /* 98 */ "wal ::= WAL INTEGER", - /* 99 */ "fsync ::= FSYNC INTEGER", - /* 100 */ "comp ::= COMP INTEGER", - /* 101 */ "prec ::= PRECISION STRING", - /* 102 */ "update ::= UPDATE INTEGER", - /* 103 */ "cachelast ::= CACHELAST INTEGER", - /* 104 */ "partitions ::= PARTITIONS INTEGER", - /* 105 */ "db_optr ::=", - /* 106 */ "db_optr ::= db_optr cache", - /* 107 */ "db_optr ::= db_optr replica", - /* 108 */ "db_optr ::= db_optr quorum", - /* 109 */ "db_optr ::= db_optr days", - /* 110 */ "db_optr ::= db_optr minrows", - /* 111 */ "db_optr ::= db_optr maxrows", - /* 112 */ "db_optr ::= db_optr blocks", - /* 113 */ "db_optr ::= db_optr ctime", - /* 114 */ "db_optr ::= db_optr wal", - /* 115 */ "db_optr ::= db_optr fsync", - /* 116 */ "db_optr ::= db_optr comp", - /* 117 */ "db_optr ::= db_optr prec", - /* 118 */ "db_optr ::= db_optr keep", - /* 119 */ "db_optr ::= db_optr update", - /* 120 */ "db_optr ::= db_optr cachelast", - /* 121 */ "topic_optr ::= db_optr", - /* 122 */ "topic_optr ::= topic_optr partitions", - /* 123 */ "alter_db_optr ::=", - /* 124 */ "alter_db_optr ::= alter_db_optr replica", - /* 125 */ "alter_db_optr ::= alter_db_optr quorum", - /* 126 */ "alter_db_optr ::= alter_db_optr keep", - /* 127 */ "alter_db_optr ::= alter_db_optr blocks", - /* 128 */ "alter_db_optr ::= alter_db_optr comp", - /* 129 */ "alter_db_optr ::= alter_db_optr update", - /* 130 */ "alter_db_optr ::= alter_db_optr cachelast", - /* 131 */ "alter_topic_optr ::= alter_db_optr", - /* 132 */ "alter_topic_optr ::= alter_topic_optr partitions", - /* 133 */ "typename ::= ids", - /* 134 */ "typename ::= ids LP signed RP", - /* 135 */ "typename ::= ids UNSIGNED", - /* 136 */ "signed ::= INTEGER", - /* 137 */ "signed ::= PLUS INTEGER", - /* 138 */ "signed ::= MINUS INTEGER", - /* 139 */ "cmd ::= CREATE TABLE create_table_args", - /* 140 */ "cmd ::= CREATE TABLE create_stable_args", - /* 141 */ "cmd ::= CREATE STABLE create_stable_args", - /* 142 */ "cmd ::= CREATE TABLE create_table_list", - /* 143 */ "create_table_list ::= create_from_stable", - /* 144 */ "create_table_list ::= create_table_list create_from_stable", - /* 145 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", - /* 146 */ "create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", - /* 147 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", - /* 148 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP", - /* 149 */ "tagNamelist ::= tagNamelist COMMA ids", - /* 150 */ "tagNamelist ::= ids", - /* 151 */ "create_table_args ::= ifnotexists ids cpxName AS select", - /* 152 */ "columnlist ::= columnlist COMMA column", - /* 153 */ "columnlist ::= column", - /* 154 */ "column ::= ids typename", - /* 155 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 156 */ "tagitemlist ::= tagitem", - /* 157 */ "tagitem ::= INTEGER", - /* 158 */ "tagitem ::= FLOAT", - /* 159 */ "tagitem ::= STRING", - /* 160 */ "tagitem ::= BOOL", - /* 161 */ "tagitem ::= NULL", - /* 162 */ "tagitem ::= NOW", - /* 163 */ "tagitem ::= MINUS INTEGER", - /* 164 */ "tagitem ::= MINUS FLOAT", - /* 165 */ "tagitem ::= PLUS INTEGER", - /* 166 */ "tagitem ::= PLUS FLOAT", - /* 167 */ "select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt", - /* 168 */ "select ::= LP select RP", - /* 169 */ "union ::= select", - /* 170 */ "union ::= union UNION ALL select", - /* 171 */ "cmd ::= union", - /* 172 */ "select ::= SELECT selcollist", - /* 173 */ "sclp ::= selcollist COMMA", - /* 174 */ "sclp ::=", - /* 175 */ "selcollist ::= sclp distinct expr as", - /* 176 */ "selcollist ::= sclp STAR", - /* 177 */ "as ::= AS ids", - /* 178 */ "as ::= ids", - /* 179 */ "as ::=", - /* 180 */ "distinct ::= DISTINCT", - /* 181 */ "distinct ::=", - /* 182 */ "from ::= FROM tablelist", - /* 183 */ "from ::= FROM sub", - /* 184 */ "sub ::= LP union RP", - /* 185 */ "sub ::= LP union RP ids", - /* 186 */ "sub ::= sub COMMA LP union RP ids", - /* 187 */ "tablelist ::= ids cpxName", - /* 188 */ "tablelist ::= ids cpxName ids", - /* 189 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 190 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 191 */ "tmvar ::= VARIABLE", - /* 192 */ "interval_option ::= intervalKey LP tmvar RP", - /* 193 */ "interval_option ::= intervalKey LP tmvar COMMA tmvar RP", - /* 194 */ "interval_option ::=", - /* 195 */ "intervalKey ::= INTERVAL", - /* 196 */ "intervalKey ::= EVERY", - /* 197 */ "session_option ::=", - /* 198 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", - /* 199 */ "windowstate_option ::=", - /* 200 */ "windowstate_option ::= STATE_WINDOW LP ids RP", - /* 201 */ "fill_opt ::=", - /* 202 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 203 */ "fill_opt ::= FILL LP ID RP", - /* 204 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 205 */ "sliding_opt ::=", - /* 206 */ "orderby_opt ::=", - /* 207 */ "orderby_opt ::= ORDER BY sortlist", - /* 208 */ "sortlist ::= sortlist COMMA item sortorder", - /* 209 */ "sortlist ::= item sortorder", - /* 210 */ "item ::= ids cpxName", - /* 211 */ "sortorder ::= ASC", - /* 212 */ "sortorder ::= DESC", - /* 213 */ "sortorder ::=", - /* 214 */ "groupby_opt ::=", - /* 215 */ "groupby_opt ::= GROUP BY grouplist", - /* 216 */ "grouplist ::= grouplist COMMA item", - /* 217 */ "grouplist ::= item", - /* 218 */ "having_opt ::=", - /* 219 */ "having_opt ::= HAVING expr", - /* 220 */ "limit_opt ::=", - /* 221 */ "limit_opt ::= LIMIT signed", - /* 222 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 223 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 224 */ "slimit_opt ::=", - /* 225 */ "slimit_opt ::= SLIMIT signed", - /* 226 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 227 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 228 */ "where_opt ::=", - /* 229 */ "where_opt ::= WHERE expr", - /* 230 */ "expr ::= LP expr RP", - /* 231 */ "expr ::= ID", - /* 232 */ "expr ::= ID DOT ID", - /* 233 */ "expr ::= ID DOT STAR", - /* 234 */ "expr ::= INTEGER", - /* 235 */ "expr ::= MINUS INTEGER", - /* 236 */ "expr ::= PLUS INTEGER", - /* 237 */ "expr ::= FLOAT", - /* 238 */ "expr ::= MINUS FLOAT", - /* 239 */ "expr ::= PLUS FLOAT", - /* 240 */ "expr ::= STRING", - /* 241 */ "expr ::= NOW", - /* 242 */ "expr ::= VARIABLE", - /* 243 */ "expr ::= PLUS VARIABLE", - /* 244 */ "expr ::= MINUS VARIABLE", - /* 245 */ "expr ::= BOOL", - /* 246 */ "expr ::= NULL", - /* 247 */ "expr ::= ID LP exprlist RP", - /* 248 */ "expr ::= ID LP STAR RP", - /* 249 */ "expr ::= expr IS NULL", - /* 250 */ "expr ::= expr IS NOT NULL", - /* 251 */ "expr ::= expr LT expr", - /* 252 */ "expr ::= expr GT expr", - /* 253 */ "expr ::= expr LE expr", - /* 254 */ "expr ::= expr GE expr", - /* 255 */ "expr ::= expr NE expr", - /* 256 */ "expr ::= expr EQ expr", - /* 257 */ "expr ::= expr BETWEEN expr AND expr", - /* 258 */ "expr ::= expr AND expr", - /* 259 */ "expr ::= expr OR expr", - /* 260 */ "expr ::= expr PLUS expr", - /* 261 */ "expr ::= expr MINUS expr", - /* 262 */ "expr ::= expr STAR expr", - /* 263 */ "expr ::= expr SLASH expr", - /* 264 */ "expr ::= expr REM expr", - /* 265 */ "expr ::= expr LIKE expr", - /* 266 */ "expr ::= expr IN LP exprlist RP", - /* 267 */ "exprlist ::= exprlist COMMA expritem", - /* 268 */ "exprlist ::= expritem", - /* 269 */ "expritem ::= expr", - /* 270 */ "expritem ::=", - /* 271 */ "cmd ::= RESET QUERY CACHE", - /* 272 */ "cmd ::= SYNCDB ids REPLICA", - /* 273 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 274 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 275 */ "cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist", - /* 276 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 277 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 278 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 279 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 280 */ "cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist", - /* 281 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", - /* 282 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", - /* 283 */ "cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist", - /* 284 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", - /* 285 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", - /* 286 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", - /* 287 */ "cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem", - /* 288 */ "cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist", - /* 289 */ "cmd ::= KILL CONNECTION INTEGER", - /* 290 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 291 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 29 */ "cmd ::= DROP TABLE ifexists ids cpxName", + /* 30 */ "cmd ::= DROP STABLE ifexists ids cpxName", + /* 31 */ "cmd ::= DROP DATABASE ifexists ids", + /* 32 */ "cmd ::= DROP TOPIC ifexists ids", + /* 33 */ "cmd ::= DROP FUNCTION ids", + /* 34 */ "cmd ::= DROP DNODE ids", + /* 35 */ "cmd ::= DROP USER ids", + /* 36 */ "cmd ::= DROP ACCOUNT ids", + /* 37 */ "cmd ::= USE ids", + /* 38 */ "cmd ::= DESCRIBE ids cpxName", + /* 39 */ "cmd ::= DESC ids cpxName", + /* 40 */ "cmd ::= ALTER USER ids PASS ids", + /* 41 */ "cmd ::= ALTER USER ids PRIVILEGE ids", + /* 42 */ "cmd ::= ALTER DNODE ids ids", + /* 43 */ "cmd ::= ALTER DNODE ids ids ids", + /* 44 */ "cmd ::= ALTER LOCAL ids", + /* 45 */ "cmd ::= ALTER LOCAL ids ids", + /* 46 */ "cmd ::= ALTER DATABASE ids alter_db_optr", + /* 47 */ "cmd ::= ALTER TOPIC ids alter_topic_optr", + /* 48 */ "cmd ::= ALTER ACCOUNT ids acct_optr", + /* 49 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", + /* 50 */ "cmd ::= COMPACT VNODES IN LP exprlist RP", + /* 51 */ "ids ::= ID", + /* 52 */ "ids ::= STRING", + /* 53 */ "ifexists ::= IF EXISTS", + /* 54 */ "ifexists ::=", + /* 55 */ "ifnotexists ::= IF NOT EXISTS", + /* 56 */ "ifnotexists ::=", + /* 57 */ "cmd ::= CREATE DNODE ids", + /* 58 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", + /* 59 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", + /* 60 */ "cmd ::= CREATE TOPIC ifnotexists ids topic_optr", + /* 61 */ "cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", + /* 62 */ "cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", + /* 63 */ "cmd ::= CREATE USER ids PASS ids", + /* 64 */ "bufsize ::=", + /* 65 */ "bufsize ::= BUFSIZE INTEGER", + /* 66 */ "pps ::=", + /* 67 */ "pps ::= PPS INTEGER", + /* 68 */ "tseries ::=", + /* 69 */ "tseries ::= TSERIES INTEGER", + /* 70 */ "dbs ::=", + /* 71 */ "dbs ::= DBS INTEGER", + /* 72 */ "streams ::=", + /* 73 */ "streams ::= STREAMS INTEGER", + /* 74 */ "storage ::=", + /* 75 */ "storage ::= STORAGE INTEGER", + /* 76 */ "qtime ::=", + /* 77 */ "qtime ::= QTIME INTEGER", + /* 78 */ "users ::=", + /* 79 */ "users ::= USERS INTEGER", + /* 80 */ "conns ::=", + /* 81 */ "conns ::= CONNS INTEGER", + /* 82 */ "state ::=", + /* 83 */ "state ::= STATE ids", + /* 84 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", + /* 85 */ "intitemlist ::= intitemlist COMMA intitem", + /* 86 */ "intitemlist ::= intitem", + /* 87 */ "intitem ::= INTEGER", + /* 88 */ "keep ::= KEEP intitemlist", + /* 89 */ "cache ::= CACHE INTEGER", + /* 90 */ "replica ::= REPLICA INTEGER", + /* 91 */ "quorum ::= QUORUM INTEGER", + /* 92 */ "days ::= DAYS INTEGER", + /* 93 */ "minrows ::= MINROWS INTEGER", + /* 94 */ "maxrows ::= MAXROWS INTEGER", + /* 95 */ "blocks ::= BLOCKS INTEGER", + /* 96 */ "ctime ::= CTIME INTEGER", + /* 97 */ "wal ::= WAL INTEGER", + /* 98 */ "fsync ::= FSYNC INTEGER", + /* 99 */ "comp ::= COMP INTEGER", + /* 100 */ "prec ::= PRECISION STRING", + /* 101 */ "update ::= UPDATE INTEGER", + /* 102 */ "cachelast ::= CACHELAST INTEGER", + /* 103 */ "partitions ::= PARTITIONS INTEGER", + /* 104 */ "db_optr ::=", + /* 105 */ "db_optr ::= db_optr cache", + /* 106 */ "db_optr ::= db_optr replica", + /* 107 */ "db_optr ::= db_optr quorum", + /* 108 */ "db_optr ::= db_optr days", + /* 109 */ "db_optr ::= db_optr minrows", + /* 110 */ "db_optr ::= db_optr maxrows", + /* 111 */ "db_optr ::= db_optr blocks", + /* 112 */ "db_optr ::= db_optr ctime", + /* 113 */ "db_optr ::= db_optr wal", + /* 114 */ "db_optr ::= db_optr fsync", + /* 115 */ "db_optr ::= db_optr comp", + /* 116 */ "db_optr ::= db_optr prec", + /* 117 */ "db_optr ::= db_optr keep", + /* 118 */ "db_optr ::= db_optr update", + /* 119 */ "db_optr ::= db_optr cachelast", + /* 120 */ "topic_optr ::= db_optr", + /* 121 */ "topic_optr ::= topic_optr partitions", + /* 122 */ "alter_db_optr ::=", + /* 123 */ "alter_db_optr ::= alter_db_optr replica", + /* 124 */ "alter_db_optr ::= alter_db_optr quorum", + /* 125 */ "alter_db_optr ::= alter_db_optr keep", + /* 126 */ "alter_db_optr ::= alter_db_optr blocks", + /* 127 */ "alter_db_optr ::= alter_db_optr comp", + /* 128 */ "alter_db_optr ::= alter_db_optr update", + /* 129 */ "alter_db_optr ::= alter_db_optr cachelast", + /* 130 */ "alter_topic_optr ::= alter_db_optr", + /* 131 */ "alter_topic_optr ::= alter_topic_optr partitions", + /* 132 */ "typename ::= ids", + /* 133 */ "typename ::= ids LP signed RP", + /* 134 */ "typename ::= ids UNSIGNED", + /* 135 */ "signed ::= INTEGER", + /* 136 */ "signed ::= PLUS INTEGER", + /* 137 */ "signed ::= MINUS INTEGER", + /* 138 */ "cmd ::= CREATE TABLE create_table_args", + /* 139 */ "cmd ::= CREATE TABLE create_stable_args", + /* 140 */ "cmd ::= CREATE STABLE create_stable_args", + /* 141 */ "cmd ::= CREATE TABLE create_table_list", + /* 142 */ "create_table_list ::= create_from_stable", + /* 143 */ "create_table_list ::= create_table_list create_from_stable", + /* 144 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", + /* 145 */ "create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", + /* 146 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", + /* 147 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP", + /* 148 */ "tagNamelist ::= tagNamelist COMMA ids", + /* 149 */ "tagNamelist ::= ids", + /* 150 */ "create_table_args ::= ifnotexists ids cpxName AS select", + /* 151 */ "columnlist ::= columnlist COMMA column", + /* 152 */ "columnlist ::= column", + /* 153 */ "column ::= ids typename", + /* 154 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 155 */ "tagitemlist ::= tagitem", + /* 156 */ "tagitem ::= INTEGER", + /* 157 */ "tagitem ::= FLOAT", + /* 158 */ "tagitem ::= STRING", + /* 159 */ "tagitem ::= BOOL", + /* 160 */ "tagitem ::= NULL", + /* 161 */ "tagitem ::= NOW", + /* 162 */ "tagitem ::= NOW PLUS VARIABLE", + /* 163 */ "tagitem ::= NOW MINUS VARIABLE", + /* 164 */ "tagitem ::= MINUS INTEGER", + /* 165 */ "tagitem ::= MINUS FLOAT", + /* 166 */ "tagitem ::= PLUS INTEGER", + /* 167 */ "tagitem ::= PLUS FLOAT", + /* 168 */ "select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt", + /* 169 */ "select ::= LP select RP", + /* 170 */ "union ::= select", + /* 171 */ "union ::= union UNION ALL select", + /* 172 */ "cmd ::= union", + /* 173 */ "select ::= SELECT selcollist", + /* 174 */ "sclp ::= selcollist COMMA", + /* 175 */ "sclp ::=", + /* 176 */ "selcollist ::= sclp distinct expr as", + /* 177 */ "selcollist ::= sclp STAR", + /* 178 */ "as ::= AS ids", + /* 179 */ "as ::= ids", + /* 180 */ "as ::=", + /* 181 */ "distinct ::= DISTINCT", + /* 182 */ "distinct ::=", + /* 183 */ "from ::= FROM tablelist", + /* 184 */ "from ::= FROM sub", + /* 185 */ "sub ::= LP union RP", + /* 186 */ "sub ::= LP union RP ids", + /* 187 */ "sub ::= sub COMMA LP union RP ids", + /* 188 */ "tablelist ::= ids cpxName", + /* 189 */ "tablelist ::= ids cpxName ids", + /* 190 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 191 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 192 */ "tmvar ::= VARIABLE", + /* 193 */ "interval_option ::= intervalKey LP tmvar RP", + /* 194 */ "interval_option ::= intervalKey LP tmvar COMMA tmvar RP", + /* 195 */ "interval_option ::=", + /* 196 */ "intervalKey ::= INTERVAL", + /* 197 */ "intervalKey ::= EVERY", + /* 198 */ "session_option ::=", + /* 199 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", + /* 200 */ "windowstate_option ::=", + /* 201 */ "windowstate_option ::= STATE_WINDOW LP ids RP", + /* 202 */ "fill_opt ::=", + /* 203 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 204 */ "fill_opt ::= FILL LP ID RP", + /* 205 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 206 */ "sliding_opt ::=", + /* 207 */ "orderby_opt ::=", + /* 208 */ "orderby_opt ::= ORDER BY sortlist", + /* 209 */ "sortlist ::= sortlist COMMA item sortorder", + /* 210 */ "sortlist ::= item sortorder", + /* 211 */ "item ::= ids cpxName", + /* 212 */ "sortorder ::= ASC", + /* 213 */ "sortorder ::= DESC", + /* 214 */ "sortorder ::=", + /* 215 */ "groupby_opt ::=", + /* 216 */ "groupby_opt ::= GROUP BY grouplist", + /* 217 */ "grouplist ::= grouplist COMMA item", + /* 218 */ "grouplist ::= item", + /* 219 */ "having_opt ::=", + /* 220 */ "having_opt ::= HAVING expr", + /* 221 */ "limit_opt ::=", + /* 222 */ "limit_opt ::= LIMIT signed", + /* 223 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 224 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 225 */ "slimit_opt ::=", + /* 226 */ "slimit_opt ::= SLIMIT signed", + /* 227 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 228 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 229 */ "where_opt ::=", + /* 230 */ "where_opt ::= WHERE expr", + /* 231 */ "expr ::= LP expr RP", + /* 232 */ "expr ::= ID", + /* 233 */ "expr ::= ID DOT ID", + /* 234 */ "expr ::= ID DOT STAR", + /* 235 */ "expr ::= INTEGER", + /* 236 */ "expr ::= MINUS INTEGER", + /* 237 */ "expr ::= PLUS INTEGER", + /* 238 */ "expr ::= FLOAT", + /* 239 */ "expr ::= MINUS FLOAT", + /* 240 */ "expr ::= PLUS FLOAT", + /* 241 */ "expr ::= STRING", + /* 242 */ "expr ::= NOW", + /* 243 */ "expr ::= VARIABLE", + /* 244 */ "expr ::= PLUS VARIABLE", + /* 245 */ "expr ::= MINUS VARIABLE", + /* 246 */ "expr ::= BOOL", + /* 247 */ "expr ::= NULL", + /* 248 */ "expr ::= ID LP exprlist RP", + /* 249 */ "expr ::= ID LP STAR RP", + /* 250 */ "expr ::= expr IS NULL", + /* 251 */ "expr ::= expr IS NOT NULL", + /* 252 */ "expr ::= expr LT expr", + /* 253 */ "expr ::= expr GT expr", + /* 254 */ "expr ::= expr LE expr", + /* 255 */ "expr ::= expr GE expr", + /* 256 */ "expr ::= expr NE expr", + /* 257 */ "expr ::= expr EQ expr", + /* 258 */ "expr ::= expr BETWEEN expr AND expr", + /* 259 */ "expr ::= expr AND expr", + /* 260 */ "expr ::= expr OR expr", + /* 261 */ "expr ::= expr PLUS expr", + /* 262 */ "expr ::= expr MINUS expr", + /* 263 */ "expr ::= expr STAR expr", + /* 264 */ "expr ::= expr SLASH expr", + /* 265 */ "expr ::= expr REM expr", + /* 266 */ "expr ::= expr LIKE expr", + /* 267 */ "expr ::= expr IN LP exprlist RP", + /* 268 */ "exprlist ::= exprlist COMMA expritem", + /* 269 */ "exprlist ::= expritem", + /* 270 */ "expritem ::= expr", + /* 271 */ "expritem ::=", + /* 272 */ "cmd ::= RESET QUERY CACHE", + /* 273 */ "cmd ::= SYNCDB ids REPLICA", + /* 274 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 275 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 276 */ "cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist", + /* 277 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 278 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 279 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 280 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 281 */ "cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist", + /* 282 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", + /* 283 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", + /* 284 */ "cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist", + /* 285 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", + /* 286 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", + /* 287 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", + /* 288 */ "cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem", + /* 289 */ "cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist", + /* 290 */ "cmd ::= KILL CONNECTION INTEGER", + /* 291 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 292 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1414,28 +1625,29 @@ static int yyGrowStack(yyParser *p){ /* Initialize a new parser that has already been allocated. */ -void ParseInit(void *yypParser){ - yyParser *pParser = (yyParser*)yypParser; +void ParseInit(void *yypRawParser ParseCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + ParseCTX_STORE #ifdef YYTRACKMAXSTACKDEPTH - pParser->yyhwm = 0; + yypParser->yyhwm = 0; #endif #if YYSTACKDEPTH<=0 - pParser->yytos = NULL; - pParser->yystack = NULL; - pParser->yystksz = 0; - if( yyGrowStack(pParser) ){ - pParser->yystack = &pParser->yystk0; - pParser->yystksz = 1; + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; } #endif #ifndef YYNOERRORRECOVERY - pParser->yyerrcnt = -1; + yypParser->yyerrcnt = -1; #endif - pParser->yytos = pParser->yystack; - pParser->yystack[0].stateno = 0; - pParser->yystack[0].major = 0; + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; #if YYSTACKDEPTH>0 - pParser->yystackEnd = &pParser->yystack[YYSTACKDEPTH-1]; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #endif } @@ -1452,11 +1664,14 @@ void ParseInit(void *yypParser){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ - yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( pParser ) ParseInit(pParser); - return pParser; +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + ParseCTX_STORE + ParseInit(yypParser ParseCTX_PARAM); + } + return (void*)yypParser; } #endif /* Parse_ENGINEALWAYSONSTACK */ @@ -1473,7 +1688,8 @@ static void yy_destructor( YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1651,13 +1867,12 @@ int ParseCoverage(FILE *out){ ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. */ -static unsigned int yy_find_shift_action( - yyParser *pParser, /* The parser */ - YYCODETYPE iLookAhead /* The look-ahead token */ +static YYACTIONTYPE yy_find_shift_action( + YYCODETYPE iLookAhead, /* The look-ahead token */ + YYACTIONTYPE stateno /* Current state number */ ){ int i; - int stateno = pParser->yytos->stateno; - + if( stateno>YY_MAX_SHIFT ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); #if defined(YYCOVERAGE) @@ -1665,15 +1880,19 @@ static unsigned int yy_find_shift_action( #endif do{ i = yy_shift_ofst[stateno]; - assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -1688,15 +1907,8 @@ static unsigned int yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -1710,6 +1922,7 @@ static unsigned int yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && i<(int)(sizeof(yy_action)/sizeof(yy_action[0])) ); return yy_action[i]; } }while(1); @@ -1719,8 +1932,8 @@ static unsigned int yy_find_shift_action( ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. */ -static int yy_find_reduce_action( - int stateno, /* Current state number */ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; @@ -1749,7 +1962,8 @@ static int yy_find_reduce_action( ** The following routine is called if the stack overflows. */ static void yyStackOverflow(yyParser *yypParser){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); @@ -1760,7 +1974,8 @@ static void yyStackOverflow(yyParser *yypParser){ ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ /******** End %stack_overflow code ********************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ + ParseARG_STORE /* Suppress warning about unused %extra_argument var */ + ParseCTX_STORE } /* @@ -1789,8 +2004,8 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ - int yyNewState, /* The new state to shift in */ - int yyMajor, /* The major token to shift in */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ ParseTOKENTYPE yyMinor /* The minor token to shift in */ ){ yyStackEntry *yytos; @@ -1820,311 +2035,606 @@ static void yy_shift( yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } yytos = yypParser->yytos; - yytos->stateno = (YYACTIONTYPE)yyNewState; - yytos->major = (YYCODETYPE)yyMajor; + yytos->stateno = yyNewState; + yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; yyTraceShift(yypParser, yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - signed char nrhs; /* Negative of the number of RHS symbols in the rule */ -} yyRuleInfo[] = { - { 197, -1 }, /* (0) program ::= cmd */ - { 198, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 198, -2 }, /* (2) cmd ::= SHOW TOPICS */ - { 198, -2 }, /* (3) cmd ::= SHOW FUNCTIONS */ - { 198, -2 }, /* (4) cmd ::= SHOW MNODES */ - { 198, -2 }, /* (5) cmd ::= SHOW DNODES */ - { 198, -2 }, /* (6) cmd ::= SHOW ACCOUNTS */ - { 198, -2 }, /* (7) cmd ::= SHOW USERS */ - { 198, -2 }, /* (8) cmd ::= SHOW MODULES */ - { 198, -2 }, /* (9) cmd ::= SHOW QUERIES */ - { 198, -2 }, /* (10) cmd ::= SHOW CONNECTIONS */ - { 198, -2 }, /* (11) cmd ::= SHOW STREAMS */ - { 198, -2 }, /* (12) cmd ::= SHOW VARIABLES */ - { 198, -2 }, /* (13) cmd ::= SHOW SCORES */ - { 198, -2 }, /* (14) cmd ::= SHOW GRANTS */ - { 198, -2 }, /* (15) cmd ::= SHOW VNODES */ - { 198, -3 }, /* (16) cmd ::= SHOW VNODES IPTOKEN */ - { 199, 0 }, /* (17) dbPrefix ::= */ - { 199, -2 }, /* (18) dbPrefix ::= ids DOT */ - { 201, 0 }, /* (19) cpxName ::= */ - { 201, -2 }, /* (20) cpxName ::= DOT ids */ - { 198, -5 }, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ - { 198, -5 }, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ - { 198, -4 }, /* (23) cmd ::= SHOW CREATE DATABASE ids */ - { 198, -3 }, /* (24) cmd ::= SHOW dbPrefix TABLES */ - { 198, -5 }, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 198, -3 }, /* (26) cmd ::= SHOW dbPrefix STABLES */ - { 198, -5 }, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 198, -3 }, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ - { 198, -4 }, /* (29) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 198, -5 }, /* (30) cmd ::= DROP TABLE ifexists ids cpxName */ - { 198, -5 }, /* (31) cmd ::= DROP STABLE ifexists ids cpxName */ - { 198, -4 }, /* (32) cmd ::= DROP DATABASE ifexists ids */ - { 198, -4 }, /* (33) cmd ::= DROP TOPIC ifexists ids */ - { 198, -3 }, /* (34) cmd ::= DROP FUNCTION ids */ - { 198, -3 }, /* (35) cmd ::= DROP DNODE ids */ - { 198, -3 }, /* (36) cmd ::= DROP USER ids */ - { 198, -3 }, /* (37) cmd ::= DROP ACCOUNT ids */ - { 198, -2 }, /* (38) cmd ::= USE ids */ - { 198, -3 }, /* (39) cmd ::= DESCRIBE ids cpxName */ - { 198, -3 }, /* (40) cmd ::= DESC ids cpxName */ - { 198, -5 }, /* (41) cmd ::= ALTER USER ids PASS ids */ - { 198, -5 }, /* (42) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 198, -4 }, /* (43) cmd ::= ALTER DNODE ids ids */ - { 198, -5 }, /* (44) cmd ::= ALTER DNODE ids ids ids */ - { 198, -3 }, /* (45) cmd ::= ALTER LOCAL ids */ - { 198, -4 }, /* (46) cmd ::= ALTER LOCAL ids ids */ - { 198, -4 }, /* (47) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 198, -4 }, /* (48) cmd ::= ALTER TOPIC ids alter_topic_optr */ - { 198, -4 }, /* (49) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 198, -6 }, /* (50) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 198, -6 }, /* (51) cmd ::= COMPACT VNODES IN LP exprlist RP */ - { 200, -1 }, /* (52) ids ::= ID */ - { 200, -1 }, /* (53) ids ::= STRING */ - { 202, -2 }, /* (54) ifexists ::= IF EXISTS */ - { 202, 0 }, /* (55) ifexists ::= */ - { 207, -3 }, /* (56) ifnotexists ::= IF NOT EXISTS */ - { 207, 0 }, /* (57) ifnotexists ::= */ - { 198, -3 }, /* (58) cmd ::= CREATE DNODE ids */ - { 198, -6 }, /* (59) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 198, -5 }, /* (60) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 198, -5 }, /* (61) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ - { 198, -8 }, /* (62) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - { 198, -9 }, /* (63) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - { 198, -5 }, /* (64) cmd ::= CREATE USER ids PASS ids */ - { 211, 0 }, /* (65) bufsize ::= */ - { 211, -2 }, /* (66) bufsize ::= BUFSIZE INTEGER */ - { 212, 0 }, /* (67) pps ::= */ - { 212, -2 }, /* (68) pps ::= PPS INTEGER */ - { 213, 0 }, /* (69) tseries ::= */ - { 213, -2 }, /* (70) tseries ::= TSERIES INTEGER */ - { 214, 0 }, /* (71) dbs ::= */ - { 214, -2 }, /* (72) dbs ::= DBS INTEGER */ - { 215, 0 }, /* (73) streams ::= */ - { 215, -2 }, /* (74) streams ::= STREAMS INTEGER */ - { 216, 0 }, /* (75) storage ::= */ - { 216, -2 }, /* (76) storage ::= STORAGE INTEGER */ - { 217, 0 }, /* (77) qtime ::= */ - { 217, -2 }, /* (78) qtime ::= QTIME INTEGER */ - { 218, 0 }, /* (79) users ::= */ - { 218, -2 }, /* (80) users ::= USERS INTEGER */ - { 219, 0 }, /* (81) conns ::= */ - { 219, -2 }, /* (82) conns ::= CONNS INTEGER */ - { 220, 0 }, /* (83) state ::= */ - { 220, -2 }, /* (84) state ::= STATE ids */ - { 205, -9 }, /* (85) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 221, -3 }, /* (86) intitemlist ::= intitemlist COMMA intitem */ - { 221, -1 }, /* (87) intitemlist ::= intitem */ - { 222, -1 }, /* (88) intitem ::= INTEGER */ - { 223, -2 }, /* (89) keep ::= KEEP intitemlist */ - { 224, -2 }, /* (90) cache ::= CACHE INTEGER */ - { 225, -2 }, /* (91) replica ::= REPLICA INTEGER */ - { 226, -2 }, /* (92) quorum ::= QUORUM INTEGER */ - { 227, -2 }, /* (93) days ::= DAYS INTEGER */ - { 228, -2 }, /* (94) minrows ::= MINROWS INTEGER */ - { 229, -2 }, /* (95) maxrows ::= MAXROWS INTEGER */ - { 230, -2 }, /* (96) blocks ::= BLOCKS INTEGER */ - { 231, -2 }, /* (97) ctime ::= CTIME INTEGER */ - { 232, -2 }, /* (98) wal ::= WAL INTEGER */ - { 233, -2 }, /* (99) fsync ::= FSYNC INTEGER */ - { 234, -2 }, /* (100) comp ::= COMP INTEGER */ - { 235, -2 }, /* (101) prec ::= PRECISION STRING */ - { 236, -2 }, /* (102) update ::= UPDATE INTEGER */ - { 237, -2 }, /* (103) cachelast ::= CACHELAST INTEGER */ - { 238, -2 }, /* (104) partitions ::= PARTITIONS INTEGER */ - { 208, 0 }, /* (105) db_optr ::= */ - { 208, -2 }, /* (106) db_optr ::= db_optr cache */ - { 208, -2 }, /* (107) db_optr ::= db_optr replica */ - { 208, -2 }, /* (108) db_optr ::= db_optr quorum */ - { 208, -2 }, /* (109) db_optr ::= db_optr days */ - { 208, -2 }, /* (110) db_optr ::= db_optr minrows */ - { 208, -2 }, /* (111) db_optr ::= db_optr maxrows */ - { 208, -2 }, /* (112) db_optr ::= db_optr blocks */ - { 208, -2 }, /* (113) db_optr ::= db_optr ctime */ - { 208, -2 }, /* (114) db_optr ::= db_optr wal */ - { 208, -2 }, /* (115) db_optr ::= db_optr fsync */ - { 208, -2 }, /* (116) db_optr ::= db_optr comp */ - { 208, -2 }, /* (117) db_optr ::= db_optr prec */ - { 208, -2 }, /* (118) db_optr ::= db_optr keep */ - { 208, -2 }, /* (119) db_optr ::= db_optr update */ - { 208, -2 }, /* (120) db_optr ::= db_optr cachelast */ - { 209, -1 }, /* (121) topic_optr ::= db_optr */ - { 209, -2 }, /* (122) topic_optr ::= topic_optr partitions */ - { 203, 0 }, /* (123) alter_db_optr ::= */ - { 203, -2 }, /* (124) alter_db_optr ::= alter_db_optr replica */ - { 203, -2 }, /* (125) alter_db_optr ::= alter_db_optr quorum */ - { 203, -2 }, /* (126) alter_db_optr ::= alter_db_optr keep */ - { 203, -2 }, /* (127) alter_db_optr ::= alter_db_optr blocks */ - { 203, -2 }, /* (128) alter_db_optr ::= alter_db_optr comp */ - { 203, -2 }, /* (129) alter_db_optr ::= alter_db_optr update */ - { 203, -2 }, /* (130) alter_db_optr ::= alter_db_optr cachelast */ - { 204, -1 }, /* (131) alter_topic_optr ::= alter_db_optr */ - { 204, -2 }, /* (132) alter_topic_optr ::= alter_topic_optr partitions */ - { 210, -1 }, /* (133) typename ::= ids */ - { 210, -4 }, /* (134) typename ::= ids LP signed RP */ - { 210, -2 }, /* (135) typename ::= ids UNSIGNED */ - { 239, -1 }, /* (136) signed ::= INTEGER */ - { 239, -2 }, /* (137) signed ::= PLUS INTEGER */ - { 239, -2 }, /* (138) signed ::= MINUS INTEGER */ - { 198, -3 }, /* (139) cmd ::= CREATE TABLE create_table_args */ - { 198, -3 }, /* (140) cmd ::= CREATE TABLE create_stable_args */ - { 198, -3 }, /* (141) cmd ::= CREATE STABLE create_stable_args */ - { 198, -3 }, /* (142) cmd ::= CREATE TABLE create_table_list */ - { 242, -1 }, /* (143) create_table_list ::= create_from_stable */ - { 242, -2 }, /* (144) create_table_list ::= create_table_list create_from_stable */ - { 240, -6 }, /* (145) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - { 241, -10 }, /* (146) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - { 243, -10 }, /* (147) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - { 243, -13 }, /* (148) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ - { 246, -3 }, /* (149) tagNamelist ::= tagNamelist COMMA ids */ - { 246, -1 }, /* (150) tagNamelist ::= ids */ - { 240, -5 }, /* (151) create_table_args ::= ifnotexists ids cpxName AS select */ - { 244, -3 }, /* (152) columnlist ::= columnlist COMMA column */ - { 244, -1 }, /* (153) columnlist ::= column */ - { 248, -2 }, /* (154) column ::= ids typename */ - { 245, -3 }, /* (155) tagitemlist ::= tagitemlist COMMA tagitem */ - { 245, -1 }, /* (156) tagitemlist ::= tagitem */ - { 249, -1 }, /* (157) tagitem ::= INTEGER */ - { 249, -1 }, /* (158) tagitem ::= FLOAT */ - { 249, -1 }, /* (159) tagitem ::= STRING */ - { 249, -1 }, /* (160) tagitem ::= BOOL */ - { 249, -1 }, /* (161) tagitem ::= NULL */ - { 249, -1 }, /* (162) tagitem ::= NOW */ - { 249, -2 }, /* (163) tagitem ::= MINUS INTEGER */ - { 249, -2 }, /* (164) tagitem ::= MINUS FLOAT */ - { 249, -2 }, /* (165) tagitem ::= PLUS INTEGER */ - { 249, -2 }, /* (166) tagitem ::= PLUS FLOAT */ - { 247, -14 }, /* (167) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ - { 247, -3 }, /* (168) select ::= LP select RP */ - { 263, -1 }, /* (169) union ::= select */ - { 263, -4 }, /* (170) union ::= union UNION ALL select */ - { 198, -1 }, /* (171) cmd ::= union */ - { 247, -2 }, /* (172) select ::= SELECT selcollist */ - { 264, -2 }, /* (173) sclp ::= selcollist COMMA */ - { 264, 0 }, /* (174) sclp ::= */ - { 250, -4 }, /* (175) selcollist ::= sclp distinct expr as */ - { 250, -2 }, /* (176) selcollist ::= sclp STAR */ - { 267, -2 }, /* (177) as ::= AS ids */ - { 267, -1 }, /* (178) as ::= ids */ - { 267, 0 }, /* (179) as ::= */ - { 265, -1 }, /* (180) distinct ::= DISTINCT */ - { 265, 0 }, /* (181) distinct ::= */ - { 251, -2 }, /* (182) from ::= FROM tablelist */ - { 251, -2 }, /* (183) from ::= FROM sub */ - { 269, -3 }, /* (184) sub ::= LP union RP */ - { 269, -4 }, /* (185) sub ::= LP union RP ids */ - { 269, -6 }, /* (186) sub ::= sub COMMA LP union RP ids */ - { 268, -2 }, /* (187) tablelist ::= ids cpxName */ - { 268, -3 }, /* (188) tablelist ::= ids cpxName ids */ - { 268, -4 }, /* (189) tablelist ::= tablelist COMMA ids cpxName */ - { 268, -5 }, /* (190) tablelist ::= tablelist COMMA ids cpxName ids */ - { 270, -1 }, /* (191) tmvar ::= VARIABLE */ - { 253, -4 }, /* (192) interval_option ::= intervalKey LP tmvar RP */ - { 253, -6 }, /* (193) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ - { 253, 0 }, /* (194) interval_option ::= */ - { 271, -1 }, /* (195) intervalKey ::= INTERVAL */ - { 271, -1 }, /* (196) intervalKey ::= EVERY */ - { 255, 0 }, /* (197) session_option ::= */ - { 255, -7 }, /* (198) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - { 256, 0 }, /* (199) windowstate_option ::= */ - { 256, -4 }, /* (200) windowstate_option ::= STATE_WINDOW LP ids RP */ - { 257, 0 }, /* (201) fill_opt ::= */ - { 257, -6 }, /* (202) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 257, -4 }, /* (203) fill_opt ::= FILL LP ID RP */ - { 254, -4 }, /* (204) sliding_opt ::= SLIDING LP tmvar RP */ - { 254, 0 }, /* (205) sliding_opt ::= */ - { 260, 0 }, /* (206) orderby_opt ::= */ - { 260, -3 }, /* (207) orderby_opt ::= ORDER BY sortlist */ - { 272, -4 }, /* (208) sortlist ::= sortlist COMMA item sortorder */ - { 272, -2 }, /* (209) sortlist ::= item sortorder */ - { 274, -2 }, /* (210) item ::= ids cpxName */ - { 275, -1 }, /* (211) sortorder ::= ASC */ - { 275, -1 }, /* (212) sortorder ::= DESC */ - { 275, 0 }, /* (213) sortorder ::= */ - { 258, 0 }, /* (214) groupby_opt ::= */ - { 258, -3 }, /* (215) groupby_opt ::= GROUP BY grouplist */ - { 276, -3 }, /* (216) grouplist ::= grouplist COMMA item */ - { 276, -1 }, /* (217) grouplist ::= item */ - { 259, 0 }, /* (218) having_opt ::= */ - { 259, -2 }, /* (219) having_opt ::= HAVING expr */ - { 262, 0 }, /* (220) limit_opt ::= */ - { 262, -2 }, /* (221) limit_opt ::= LIMIT signed */ - { 262, -4 }, /* (222) limit_opt ::= LIMIT signed OFFSET signed */ - { 262, -4 }, /* (223) limit_opt ::= LIMIT signed COMMA signed */ - { 261, 0 }, /* (224) slimit_opt ::= */ - { 261, -2 }, /* (225) slimit_opt ::= SLIMIT signed */ - { 261, -4 }, /* (226) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 261, -4 }, /* (227) slimit_opt ::= SLIMIT signed COMMA signed */ - { 252, 0 }, /* (228) where_opt ::= */ - { 252, -2 }, /* (229) where_opt ::= WHERE expr */ - { 266, -3 }, /* (230) expr ::= LP expr RP */ - { 266, -1 }, /* (231) expr ::= ID */ - { 266, -3 }, /* (232) expr ::= ID DOT ID */ - { 266, -3 }, /* (233) expr ::= ID DOT STAR */ - { 266, -1 }, /* (234) expr ::= INTEGER */ - { 266, -2 }, /* (235) expr ::= MINUS INTEGER */ - { 266, -2 }, /* (236) expr ::= PLUS INTEGER */ - { 266, -1 }, /* (237) expr ::= FLOAT */ - { 266, -2 }, /* (238) expr ::= MINUS FLOAT */ - { 266, -2 }, /* (239) expr ::= PLUS FLOAT */ - { 266, -1 }, /* (240) expr ::= STRING */ - { 266, -1 }, /* (241) expr ::= NOW */ - { 266, -1 }, /* (242) expr ::= VARIABLE */ - { 266, -2 }, /* (243) expr ::= PLUS VARIABLE */ - { 266, -2 }, /* (244) expr ::= MINUS VARIABLE */ - { 266, -1 }, /* (245) expr ::= BOOL */ - { 266, -1 }, /* (246) expr ::= NULL */ - { 266, -4 }, /* (247) expr ::= ID LP exprlist RP */ - { 266, -4 }, /* (248) expr ::= ID LP STAR RP */ - { 266, -3 }, /* (249) expr ::= expr IS NULL */ - { 266, -4 }, /* (250) expr ::= expr IS NOT NULL */ - { 266, -3 }, /* (251) expr ::= expr LT expr */ - { 266, -3 }, /* (252) expr ::= expr GT expr */ - { 266, -3 }, /* (253) expr ::= expr LE expr */ - { 266, -3 }, /* (254) expr ::= expr GE expr */ - { 266, -3 }, /* (255) expr ::= expr NE expr */ - { 266, -3 }, /* (256) expr ::= expr EQ expr */ - { 266, -5 }, /* (257) expr ::= expr BETWEEN expr AND expr */ - { 266, -3 }, /* (258) expr ::= expr AND expr */ - { 266, -3 }, /* (259) expr ::= expr OR expr */ - { 266, -3 }, /* (260) expr ::= expr PLUS expr */ - { 266, -3 }, /* (261) expr ::= expr MINUS expr */ - { 266, -3 }, /* (262) expr ::= expr STAR expr */ - { 266, -3 }, /* (263) expr ::= expr SLASH expr */ - { 266, -3 }, /* (264) expr ::= expr REM expr */ - { 266, -3 }, /* (265) expr ::= expr LIKE expr */ - { 266, -5 }, /* (266) expr ::= expr IN LP exprlist RP */ - { 206, -3 }, /* (267) exprlist ::= exprlist COMMA expritem */ - { 206, -1 }, /* (268) exprlist ::= expritem */ - { 277, -1 }, /* (269) expritem ::= expr */ - { 277, 0 }, /* (270) expritem ::= */ - { 198, -3 }, /* (271) cmd ::= RESET QUERY CACHE */ - { 198, -3 }, /* (272) cmd ::= SYNCDB ids REPLICA */ - { 198, -7 }, /* (273) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 198, -7 }, /* (274) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 198, -7 }, /* (275) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ - { 198, -7 }, /* (276) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 198, -7 }, /* (277) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 198, -8 }, /* (278) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 198, -9 }, /* (279) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 198, -7 }, /* (280) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ - { 198, -7 }, /* (281) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - { 198, -7 }, /* (282) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - { 198, -7 }, /* (283) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ - { 198, -7 }, /* (284) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - { 198, -7 }, /* (285) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - { 198, -8 }, /* (286) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - { 198, -9 }, /* (287) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ - { 198, -7 }, /* (288) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ - { 198, -3 }, /* (289) cmd ::= KILL CONNECTION INTEGER */ - { 198, -5 }, /* (290) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 198, -5 }, /* (291) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { + 197, /* (0) program ::= cmd */ + 198, /* (1) cmd ::= SHOW DATABASES */ + 198, /* (2) cmd ::= SHOW TOPICS */ + 198, /* (3) cmd ::= SHOW FUNCTIONS */ + 198, /* (4) cmd ::= SHOW MNODES */ + 198, /* (5) cmd ::= SHOW DNODES */ + 198, /* (6) cmd ::= SHOW ACCOUNTS */ + 198, /* (7) cmd ::= SHOW USERS */ + 198, /* (8) cmd ::= SHOW MODULES */ + 198, /* (9) cmd ::= SHOW QUERIES */ + 198, /* (10) cmd ::= SHOW CONNECTIONS */ + 198, /* (11) cmd ::= SHOW STREAMS */ + 198, /* (12) cmd ::= SHOW VARIABLES */ + 198, /* (13) cmd ::= SHOW SCORES */ + 198, /* (14) cmd ::= SHOW GRANTS */ + 198, /* (15) cmd ::= SHOW VNODES */ + 198, /* (16) cmd ::= SHOW VNODES IPTOKEN */ + 199, /* (17) dbPrefix ::= */ + 199, /* (18) dbPrefix ::= ids DOT */ + 201, /* (19) cpxName ::= */ + 201, /* (20) cpxName ::= DOT ids */ + 198, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ + 198, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ + 198, /* (23) cmd ::= SHOW CREATE DATABASE ids */ + 198, /* (24) cmd ::= SHOW dbPrefix TABLES */ + 198, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + 198, /* (26) cmd ::= SHOW dbPrefix STABLES */ + 198, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + 198, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ + 198, /* (29) cmd ::= DROP TABLE ifexists ids cpxName */ + 198, /* (30) cmd ::= DROP STABLE ifexists ids cpxName */ + 198, /* (31) cmd ::= DROP DATABASE ifexists ids */ + 198, /* (32) cmd ::= DROP TOPIC ifexists ids */ + 198, /* (33) cmd ::= DROP FUNCTION ids */ + 198, /* (34) cmd ::= DROP DNODE ids */ + 198, /* (35) cmd ::= DROP USER ids */ + 198, /* (36) cmd ::= DROP ACCOUNT ids */ + 198, /* (37) cmd ::= USE ids */ + 198, /* (38) cmd ::= DESCRIBE ids cpxName */ + 198, /* (39) cmd ::= DESC ids cpxName */ + 198, /* (40) cmd ::= ALTER USER ids PASS ids */ + 198, /* (41) cmd ::= ALTER USER ids PRIVILEGE ids */ + 198, /* (42) cmd ::= ALTER DNODE ids ids */ + 198, /* (43) cmd ::= ALTER DNODE ids ids ids */ + 198, /* (44) cmd ::= ALTER LOCAL ids */ + 198, /* (45) cmd ::= ALTER LOCAL ids ids */ + 198, /* (46) cmd ::= ALTER DATABASE ids alter_db_optr */ + 198, /* (47) cmd ::= ALTER TOPIC ids alter_topic_optr */ + 198, /* (48) cmd ::= ALTER ACCOUNT ids acct_optr */ + 198, /* (49) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + 198, /* (50) cmd ::= COMPACT VNODES IN LP exprlist RP */ + 200, /* (51) ids ::= ID */ + 200, /* (52) ids ::= STRING */ + 202, /* (53) ifexists ::= IF EXISTS */ + 202, /* (54) ifexists ::= */ + 207, /* (55) ifnotexists ::= IF NOT EXISTS */ + 207, /* (56) ifnotexists ::= */ + 198, /* (57) cmd ::= CREATE DNODE ids */ + 198, /* (58) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + 198, /* (59) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + 198, /* (60) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ + 198, /* (61) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + 198, /* (62) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + 198, /* (63) cmd ::= CREATE USER ids PASS ids */ + 211, /* (64) bufsize ::= */ + 211, /* (65) bufsize ::= BUFSIZE INTEGER */ + 212, /* (66) pps ::= */ + 212, /* (67) pps ::= PPS INTEGER */ + 213, /* (68) tseries ::= */ + 213, /* (69) tseries ::= TSERIES INTEGER */ + 214, /* (70) dbs ::= */ + 214, /* (71) dbs ::= DBS INTEGER */ + 215, /* (72) streams ::= */ + 215, /* (73) streams ::= STREAMS INTEGER */ + 216, /* (74) storage ::= */ + 216, /* (75) storage ::= STORAGE INTEGER */ + 217, /* (76) qtime ::= */ + 217, /* (77) qtime ::= QTIME INTEGER */ + 218, /* (78) users ::= */ + 218, /* (79) users ::= USERS INTEGER */ + 219, /* (80) conns ::= */ + 219, /* (81) conns ::= CONNS INTEGER */ + 220, /* (82) state ::= */ + 220, /* (83) state ::= STATE ids */ + 205, /* (84) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + 221, /* (85) intitemlist ::= intitemlist COMMA intitem */ + 221, /* (86) intitemlist ::= intitem */ + 222, /* (87) intitem ::= INTEGER */ + 223, /* (88) keep ::= KEEP intitemlist */ + 224, /* (89) cache ::= CACHE INTEGER */ + 225, /* (90) replica ::= REPLICA INTEGER */ + 226, /* (91) quorum ::= QUORUM INTEGER */ + 227, /* (92) days ::= DAYS INTEGER */ + 228, /* (93) minrows ::= MINROWS INTEGER */ + 229, /* (94) maxrows ::= MAXROWS INTEGER */ + 230, /* (95) blocks ::= BLOCKS INTEGER */ + 231, /* (96) ctime ::= CTIME INTEGER */ + 232, /* (97) wal ::= WAL INTEGER */ + 233, /* (98) fsync ::= FSYNC INTEGER */ + 234, /* (99) comp ::= COMP INTEGER */ + 235, /* (100) prec ::= PRECISION STRING */ + 236, /* (101) update ::= UPDATE INTEGER */ + 237, /* (102) cachelast ::= CACHELAST INTEGER */ + 238, /* (103) partitions ::= PARTITIONS INTEGER */ + 208, /* (104) db_optr ::= */ + 208, /* (105) db_optr ::= db_optr cache */ + 208, /* (106) db_optr ::= db_optr replica */ + 208, /* (107) db_optr ::= db_optr quorum */ + 208, /* (108) db_optr ::= db_optr days */ + 208, /* (109) db_optr ::= db_optr minrows */ + 208, /* (110) db_optr ::= db_optr maxrows */ + 208, /* (111) db_optr ::= db_optr blocks */ + 208, /* (112) db_optr ::= db_optr ctime */ + 208, /* (113) db_optr ::= db_optr wal */ + 208, /* (114) db_optr ::= db_optr fsync */ + 208, /* (115) db_optr ::= db_optr comp */ + 208, /* (116) db_optr ::= db_optr prec */ + 208, /* (117) db_optr ::= db_optr keep */ + 208, /* (118) db_optr ::= db_optr update */ + 208, /* (119) db_optr ::= db_optr cachelast */ + 209, /* (120) topic_optr ::= db_optr */ + 209, /* (121) topic_optr ::= topic_optr partitions */ + 203, /* (122) alter_db_optr ::= */ + 203, /* (123) alter_db_optr ::= alter_db_optr replica */ + 203, /* (124) alter_db_optr ::= alter_db_optr quorum */ + 203, /* (125) alter_db_optr ::= alter_db_optr keep */ + 203, /* (126) alter_db_optr ::= alter_db_optr blocks */ + 203, /* (127) alter_db_optr ::= alter_db_optr comp */ + 203, /* (128) alter_db_optr ::= alter_db_optr update */ + 203, /* (129) alter_db_optr ::= alter_db_optr cachelast */ + 204, /* (130) alter_topic_optr ::= alter_db_optr */ + 204, /* (131) alter_topic_optr ::= alter_topic_optr partitions */ + 210, /* (132) typename ::= ids */ + 210, /* (133) typename ::= ids LP signed RP */ + 210, /* (134) typename ::= ids UNSIGNED */ + 239, /* (135) signed ::= INTEGER */ + 239, /* (136) signed ::= PLUS INTEGER */ + 239, /* (137) signed ::= MINUS INTEGER */ + 198, /* (138) cmd ::= CREATE TABLE create_table_args */ + 198, /* (139) cmd ::= CREATE TABLE create_stable_args */ + 198, /* (140) cmd ::= CREATE STABLE create_stable_args */ + 198, /* (141) cmd ::= CREATE TABLE create_table_list */ + 242, /* (142) create_table_list ::= create_from_stable */ + 242, /* (143) create_table_list ::= create_table_list create_from_stable */ + 240, /* (144) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + 241, /* (145) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + 243, /* (146) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + 243, /* (147) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ + 246, /* (148) tagNamelist ::= tagNamelist COMMA ids */ + 246, /* (149) tagNamelist ::= ids */ + 240, /* (150) create_table_args ::= ifnotexists ids cpxName AS select */ + 244, /* (151) columnlist ::= columnlist COMMA column */ + 244, /* (152) columnlist ::= column */ + 248, /* (153) column ::= ids typename */ + 245, /* (154) tagitemlist ::= tagitemlist COMMA tagitem */ + 245, /* (155) tagitemlist ::= tagitem */ + 249, /* (156) tagitem ::= INTEGER */ + 249, /* (157) tagitem ::= FLOAT */ + 249, /* (158) tagitem ::= STRING */ + 249, /* (159) tagitem ::= BOOL */ + 249, /* (160) tagitem ::= NULL */ + 249, /* (161) tagitem ::= NOW */ + 249, /* (162) tagitem ::= NOW PLUS VARIABLE */ + 249, /* (163) tagitem ::= NOW MINUS VARIABLE */ + 249, /* (164) tagitem ::= MINUS INTEGER */ + 249, /* (165) tagitem ::= MINUS FLOAT */ + 249, /* (166) tagitem ::= PLUS INTEGER */ + 249, /* (167) tagitem ::= PLUS FLOAT */ + 247, /* (168) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ + 247, /* (169) select ::= LP select RP */ + 263, /* (170) union ::= select */ + 263, /* (171) union ::= union UNION ALL select */ + 198, /* (172) cmd ::= union */ + 247, /* (173) select ::= SELECT selcollist */ + 264, /* (174) sclp ::= selcollist COMMA */ + 264, /* (175) sclp ::= */ + 250, /* (176) selcollist ::= sclp distinct expr as */ + 250, /* (177) selcollist ::= sclp STAR */ + 267, /* (178) as ::= AS ids */ + 267, /* (179) as ::= ids */ + 267, /* (180) as ::= */ + 265, /* (181) distinct ::= DISTINCT */ + 265, /* (182) distinct ::= */ + 251, /* (183) from ::= FROM tablelist */ + 251, /* (184) from ::= FROM sub */ + 269, /* (185) sub ::= LP union RP */ + 269, /* (186) sub ::= LP union RP ids */ + 269, /* (187) sub ::= sub COMMA LP union RP ids */ + 268, /* (188) tablelist ::= ids cpxName */ + 268, /* (189) tablelist ::= ids cpxName ids */ + 268, /* (190) tablelist ::= tablelist COMMA ids cpxName */ + 268, /* (191) tablelist ::= tablelist COMMA ids cpxName ids */ + 270, /* (192) tmvar ::= VARIABLE */ + 253, /* (193) interval_option ::= intervalKey LP tmvar RP */ + 253, /* (194) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ + 253, /* (195) interval_option ::= */ + 271, /* (196) intervalKey ::= INTERVAL */ + 271, /* (197) intervalKey ::= EVERY */ + 255, /* (198) session_option ::= */ + 255, /* (199) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + 256, /* (200) windowstate_option ::= */ + 256, /* (201) windowstate_option ::= STATE_WINDOW LP ids RP */ + 257, /* (202) fill_opt ::= */ + 257, /* (203) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + 257, /* (204) fill_opt ::= FILL LP ID RP */ + 254, /* (205) sliding_opt ::= SLIDING LP tmvar RP */ + 254, /* (206) sliding_opt ::= */ + 260, /* (207) orderby_opt ::= */ + 260, /* (208) orderby_opt ::= ORDER BY sortlist */ + 272, /* (209) sortlist ::= sortlist COMMA item sortorder */ + 272, /* (210) sortlist ::= item sortorder */ + 274, /* (211) item ::= ids cpxName */ + 275, /* (212) sortorder ::= ASC */ + 275, /* (213) sortorder ::= DESC */ + 275, /* (214) sortorder ::= */ + 258, /* (215) groupby_opt ::= */ + 258, /* (216) groupby_opt ::= GROUP BY grouplist */ + 276, /* (217) grouplist ::= grouplist COMMA item */ + 276, /* (218) grouplist ::= item */ + 259, /* (219) having_opt ::= */ + 259, /* (220) having_opt ::= HAVING expr */ + 262, /* (221) limit_opt ::= */ + 262, /* (222) limit_opt ::= LIMIT signed */ + 262, /* (223) limit_opt ::= LIMIT signed OFFSET signed */ + 262, /* (224) limit_opt ::= LIMIT signed COMMA signed */ + 261, /* (225) slimit_opt ::= */ + 261, /* (226) slimit_opt ::= SLIMIT signed */ + 261, /* (227) slimit_opt ::= SLIMIT signed SOFFSET signed */ + 261, /* (228) slimit_opt ::= SLIMIT signed COMMA signed */ + 252, /* (229) where_opt ::= */ + 252, /* (230) where_opt ::= WHERE expr */ + 266, /* (231) expr ::= LP expr RP */ + 266, /* (232) expr ::= ID */ + 266, /* (233) expr ::= ID DOT ID */ + 266, /* (234) expr ::= ID DOT STAR */ + 266, /* (235) expr ::= INTEGER */ + 266, /* (236) expr ::= MINUS INTEGER */ + 266, /* (237) expr ::= PLUS INTEGER */ + 266, /* (238) expr ::= FLOAT */ + 266, /* (239) expr ::= MINUS FLOAT */ + 266, /* (240) expr ::= PLUS FLOAT */ + 266, /* (241) expr ::= STRING */ + 266, /* (242) expr ::= NOW */ + 266, /* (243) expr ::= VARIABLE */ + 266, /* (244) expr ::= PLUS VARIABLE */ + 266, /* (245) expr ::= MINUS VARIABLE */ + 266, /* (246) expr ::= BOOL */ + 266, /* (247) expr ::= NULL */ + 266, /* (248) expr ::= ID LP exprlist RP */ + 266, /* (249) expr ::= ID LP STAR RP */ + 266, /* (250) expr ::= expr IS NULL */ + 266, /* (251) expr ::= expr IS NOT NULL */ + 266, /* (252) expr ::= expr LT expr */ + 266, /* (253) expr ::= expr GT expr */ + 266, /* (254) expr ::= expr LE expr */ + 266, /* (255) expr ::= expr GE expr */ + 266, /* (256) expr ::= expr NE expr */ + 266, /* (257) expr ::= expr EQ expr */ + 266, /* (258) expr ::= expr BETWEEN expr AND expr */ + 266, /* (259) expr ::= expr AND expr */ + 266, /* (260) expr ::= expr OR expr */ + 266, /* (261) expr ::= expr PLUS expr */ + 266, /* (262) expr ::= expr MINUS expr */ + 266, /* (263) expr ::= expr STAR expr */ + 266, /* (264) expr ::= expr SLASH expr */ + 266, /* (265) expr ::= expr REM expr */ + 266, /* (266) expr ::= expr LIKE expr */ + 266, /* (267) expr ::= expr IN LP exprlist RP */ + 206, /* (268) exprlist ::= exprlist COMMA expritem */ + 206, /* (269) exprlist ::= expritem */ + 277, /* (270) expritem ::= expr */ + 277, /* (271) expritem ::= */ + 198, /* (272) cmd ::= RESET QUERY CACHE */ + 198, /* (273) cmd ::= SYNCDB ids REPLICA */ + 198, /* (274) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + 198, /* (275) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + 198, /* (276) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ + 198, /* (277) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + 198, /* (278) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + 198, /* (279) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + 198, /* (280) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + 198, /* (281) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ + 198, /* (282) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + 198, /* (283) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + 198, /* (284) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ + 198, /* (285) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + 198, /* (286) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + 198, /* (287) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + 198, /* (288) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ + 198, /* (289) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ + 198, /* (290) cmd ::= KILL CONNECTION INTEGER */ + 198, /* (291) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + 198, /* (292) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { + -1, /* (0) program ::= cmd */ + -2, /* (1) cmd ::= SHOW DATABASES */ + -2, /* (2) cmd ::= SHOW TOPICS */ + -2, /* (3) cmd ::= SHOW FUNCTIONS */ + -2, /* (4) cmd ::= SHOW MNODES */ + -2, /* (5) cmd ::= SHOW DNODES */ + -2, /* (6) cmd ::= SHOW ACCOUNTS */ + -2, /* (7) cmd ::= SHOW USERS */ + -2, /* (8) cmd ::= SHOW MODULES */ + -2, /* (9) cmd ::= SHOW QUERIES */ + -2, /* (10) cmd ::= SHOW CONNECTIONS */ + -2, /* (11) cmd ::= SHOW STREAMS */ + -2, /* (12) cmd ::= SHOW VARIABLES */ + -2, /* (13) cmd ::= SHOW SCORES */ + -2, /* (14) cmd ::= SHOW GRANTS */ + -2, /* (15) cmd ::= SHOW VNODES */ + -3, /* (16) cmd ::= SHOW VNODES IPTOKEN */ + 0, /* (17) dbPrefix ::= */ + -2, /* (18) dbPrefix ::= ids DOT */ + 0, /* (19) cpxName ::= */ + -2, /* (20) cpxName ::= DOT ids */ + -5, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ + -5, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ + -4, /* (23) cmd ::= SHOW CREATE DATABASE ids */ + -3, /* (24) cmd ::= SHOW dbPrefix TABLES */ + -5, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + -3, /* (26) cmd ::= SHOW dbPrefix STABLES */ + -5, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + -3, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ + -5, /* (29) cmd ::= DROP TABLE ifexists ids cpxName */ + -5, /* (30) cmd ::= DROP STABLE ifexists ids cpxName */ + -4, /* (31) cmd ::= DROP DATABASE ifexists ids */ + -4, /* (32) cmd ::= DROP TOPIC ifexists ids */ + -3, /* (33) cmd ::= DROP FUNCTION ids */ + -3, /* (34) cmd ::= DROP DNODE ids */ + -3, /* (35) cmd ::= DROP USER ids */ + -3, /* (36) cmd ::= DROP ACCOUNT ids */ + -2, /* (37) cmd ::= USE ids */ + -3, /* (38) cmd ::= DESCRIBE ids cpxName */ + -3, /* (39) cmd ::= DESC ids cpxName */ + -5, /* (40) cmd ::= ALTER USER ids PASS ids */ + -5, /* (41) cmd ::= ALTER USER ids PRIVILEGE ids */ + -4, /* (42) cmd ::= ALTER DNODE ids ids */ + -5, /* (43) cmd ::= ALTER DNODE ids ids ids */ + -3, /* (44) cmd ::= ALTER LOCAL ids */ + -4, /* (45) cmd ::= ALTER LOCAL ids ids */ + -4, /* (46) cmd ::= ALTER DATABASE ids alter_db_optr */ + -4, /* (47) cmd ::= ALTER TOPIC ids alter_topic_optr */ + -4, /* (48) cmd ::= ALTER ACCOUNT ids acct_optr */ + -6, /* (49) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + -6, /* (50) cmd ::= COMPACT VNODES IN LP exprlist RP */ + -1, /* (51) ids ::= ID */ + -1, /* (52) ids ::= STRING */ + -2, /* (53) ifexists ::= IF EXISTS */ + 0, /* (54) ifexists ::= */ + -3, /* (55) ifnotexists ::= IF NOT EXISTS */ + 0, /* (56) ifnotexists ::= */ + -3, /* (57) cmd ::= CREATE DNODE ids */ + -6, /* (58) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + -5, /* (59) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + -5, /* (60) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ + -8, /* (61) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + -9, /* (62) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + -5, /* (63) cmd ::= CREATE USER ids PASS ids */ + 0, /* (64) bufsize ::= */ + -2, /* (65) bufsize ::= BUFSIZE INTEGER */ + 0, /* (66) pps ::= */ + -2, /* (67) pps ::= PPS INTEGER */ + 0, /* (68) tseries ::= */ + -2, /* (69) tseries ::= TSERIES INTEGER */ + 0, /* (70) dbs ::= */ + -2, /* (71) dbs ::= DBS INTEGER */ + 0, /* (72) streams ::= */ + -2, /* (73) streams ::= STREAMS INTEGER */ + 0, /* (74) storage ::= */ + -2, /* (75) storage ::= STORAGE INTEGER */ + 0, /* (76) qtime ::= */ + -2, /* (77) qtime ::= QTIME INTEGER */ + 0, /* (78) users ::= */ + -2, /* (79) users ::= USERS INTEGER */ + 0, /* (80) conns ::= */ + -2, /* (81) conns ::= CONNS INTEGER */ + 0, /* (82) state ::= */ + -2, /* (83) state ::= STATE ids */ + -9, /* (84) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + -3, /* (85) intitemlist ::= intitemlist COMMA intitem */ + -1, /* (86) intitemlist ::= intitem */ + -1, /* (87) intitem ::= INTEGER */ + -2, /* (88) keep ::= KEEP intitemlist */ + -2, /* (89) cache ::= CACHE INTEGER */ + -2, /* (90) replica ::= REPLICA INTEGER */ + -2, /* (91) quorum ::= QUORUM INTEGER */ + -2, /* (92) days ::= DAYS INTEGER */ + -2, /* (93) minrows ::= MINROWS INTEGER */ + -2, /* (94) maxrows ::= MAXROWS INTEGER */ + -2, /* (95) blocks ::= BLOCKS INTEGER */ + -2, /* (96) ctime ::= CTIME INTEGER */ + -2, /* (97) wal ::= WAL INTEGER */ + -2, /* (98) fsync ::= FSYNC INTEGER */ + -2, /* (99) comp ::= COMP INTEGER */ + -2, /* (100) prec ::= PRECISION STRING */ + -2, /* (101) update ::= UPDATE INTEGER */ + -2, /* (102) cachelast ::= CACHELAST INTEGER */ + -2, /* (103) partitions ::= PARTITIONS INTEGER */ + 0, /* (104) db_optr ::= */ + -2, /* (105) db_optr ::= db_optr cache */ + -2, /* (106) db_optr ::= db_optr replica */ + -2, /* (107) db_optr ::= db_optr quorum */ + -2, /* (108) db_optr ::= db_optr days */ + -2, /* (109) db_optr ::= db_optr minrows */ + -2, /* (110) db_optr ::= db_optr maxrows */ + -2, /* (111) db_optr ::= db_optr blocks */ + -2, /* (112) db_optr ::= db_optr ctime */ + -2, /* (113) db_optr ::= db_optr wal */ + -2, /* (114) db_optr ::= db_optr fsync */ + -2, /* (115) db_optr ::= db_optr comp */ + -2, /* (116) db_optr ::= db_optr prec */ + -2, /* (117) db_optr ::= db_optr keep */ + -2, /* (118) db_optr ::= db_optr update */ + -2, /* (119) db_optr ::= db_optr cachelast */ + -1, /* (120) topic_optr ::= db_optr */ + -2, /* (121) topic_optr ::= topic_optr partitions */ + 0, /* (122) alter_db_optr ::= */ + -2, /* (123) alter_db_optr ::= alter_db_optr replica */ + -2, /* (124) alter_db_optr ::= alter_db_optr quorum */ + -2, /* (125) alter_db_optr ::= alter_db_optr keep */ + -2, /* (126) alter_db_optr ::= alter_db_optr blocks */ + -2, /* (127) alter_db_optr ::= alter_db_optr comp */ + -2, /* (128) alter_db_optr ::= alter_db_optr update */ + -2, /* (129) alter_db_optr ::= alter_db_optr cachelast */ + -1, /* (130) alter_topic_optr ::= alter_db_optr */ + -2, /* (131) alter_topic_optr ::= alter_topic_optr partitions */ + -1, /* (132) typename ::= ids */ + -4, /* (133) typename ::= ids LP signed RP */ + -2, /* (134) typename ::= ids UNSIGNED */ + -1, /* (135) signed ::= INTEGER */ + -2, /* (136) signed ::= PLUS INTEGER */ + -2, /* (137) signed ::= MINUS INTEGER */ + -3, /* (138) cmd ::= CREATE TABLE create_table_args */ + -3, /* (139) cmd ::= CREATE TABLE create_stable_args */ + -3, /* (140) cmd ::= CREATE STABLE create_stable_args */ + -3, /* (141) cmd ::= CREATE TABLE create_table_list */ + -1, /* (142) create_table_list ::= create_from_stable */ + -2, /* (143) create_table_list ::= create_table_list create_from_stable */ + -6, /* (144) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + -10, /* (145) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + -10, /* (146) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + -13, /* (147) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ + -3, /* (148) tagNamelist ::= tagNamelist COMMA ids */ + -1, /* (149) tagNamelist ::= ids */ + -5, /* (150) create_table_args ::= ifnotexists ids cpxName AS select */ + -3, /* (151) columnlist ::= columnlist COMMA column */ + -1, /* (152) columnlist ::= column */ + -2, /* (153) column ::= ids typename */ + -3, /* (154) tagitemlist ::= tagitemlist COMMA tagitem */ + -1, /* (155) tagitemlist ::= tagitem */ + -1, /* (156) tagitem ::= INTEGER */ + -1, /* (157) tagitem ::= FLOAT */ + -1, /* (158) tagitem ::= STRING */ + -1, /* (159) tagitem ::= BOOL */ + -1, /* (160) tagitem ::= NULL */ + -1, /* (161) tagitem ::= NOW */ + -3, /* (162) tagitem ::= NOW PLUS VARIABLE */ + -3, /* (163) tagitem ::= NOW MINUS VARIABLE */ + -2, /* (164) tagitem ::= MINUS INTEGER */ + -2, /* (165) tagitem ::= MINUS FLOAT */ + -2, /* (166) tagitem ::= PLUS INTEGER */ + -2, /* (167) tagitem ::= PLUS FLOAT */ + -14, /* (168) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ + -3, /* (169) select ::= LP select RP */ + -1, /* (170) union ::= select */ + -4, /* (171) union ::= union UNION ALL select */ + -1, /* (172) cmd ::= union */ + -2, /* (173) select ::= SELECT selcollist */ + -2, /* (174) sclp ::= selcollist COMMA */ + 0, /* (175) sclp ::= */ + -4, /* (176) selcollist ::= sclp distinct expr as */ + -2, /* (177) selcollist ::= sclp STAR */ + -2, /* (178) as ::= AS ids */ + -1, /* (179) as ::= ids */ + 0, /* (180) as ::= */ + -1, /* (181) distinct ::= DISTINCT */ + 0, /* (182) distinct ::= */ + -2, /* (183) from ::= FROM tablelist */ + -2, /* (184) from ::= FROM sub */ + -3, /* (185) sub ::= LP union RP */ + -4, /* (186) sub ::= LP union RP ids */ + -6, /* (187) sub ::= sub COMMA LP union RP ids */ + -2, /* (188) tablelist ::= ids cpxName */ + -3, /* (189) tablelist ::= ids cpxName ids */ + -4, /* (190) tablelist ::= tablelist COMMA ids cpxName */ + -5, /* (191) tablelist ::= tablelist COMMA ids cpxName ids */ + -1, /* (192) tmvar ::= VARIABLE */ + -4, /* (193) interval_option ::= intervalKey LP tmvar RP */ + -6, /* (194) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ + 0, /* (195) interval_option ::= */ + -1, /* (196) intervalKey ::= INTERVAL */ + -1, /* (197) intervalKey ::= EVERY */ + 0, /* (198) session_option ::= */ + -7, /* (199) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + 0, /* (200) windowstate_option ::= */ + -4, /* (201) windowstate_option ::= STATE_WINDOW LP ids RP */ + 0, /* (202) fill_opt ::= */ + -6, /* (203) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + -4, /* (204) fill_opt ::= FILL LP ID RP */ + -4, /* (205) sliding_opt ::= SLIDING LP tmvar RP */ + 0, /* (206) sliding_opt ::= */ + 0, /* (207) orderby_opt ::= */ + -3, /* (208) orderby_opt ::= ORDER BY sortlist */ + -4, /* (209) sortlist ::= sortlist COMMA item sortorder */ + -2, /* (210) sortlist ::= item sortorder */ + -2, /* (211) item ::= ids cpxName */ + -1, /* (212) sortorder ::= ASC */ + -1, /* (213) sortorder ::= DESC */ + 0, /* (214) sortorder ::= */ + 0, /* (215) groupby_opt ::= */ + -3, /* (216) groupby_opt ::= GROUP BY grouplist */ + -3, /* (217) grouplist ::= grouplist COMMA item */ + -1, /* (218) grouplist ::= item */ + 0, /* (219) having_opt ::= */ + -2, /* (220) having_opt ::= HAVING expr */ + 0, /* (221) limit_opt ::= */ + -2, /* (222) limit_opt ::= LIMIT signed */ + -4, /* (223) limit_opt ::= LIMIT signed OFFSET signed */ + -4, /* (224) limit_opt ::= LIMIT signed COMMA signed */ + 0, /* (225) slimit_opt ::= */ + -2, /* (226) slimit_opt ::= SLIMIT signed */ + -4, /* (227) slimit_opt ::= SLIMIT signed SOFFSET signed */ + -4, /* (228) slimit_opt ::= SLIMIT signed COMMA signed */ + 0, /* (229) where_opt ::= */ + -2, /* (230) where_opt ::= WHERE expr */ + -3, /* (231) expr ::= LP expr RP */ + -1, /* (232) expr ::= ID */ + -3, /* (233) expr ::= ID DOT ID */ + -3, /* (234) expr ::= ID DOT STAR */ + -1, /* (235) expr ::= INTEGER */ + -2, /* (236) expr ::= MINUS INTEGER */ + -2, /* (237) expr ::= PLUS INTEGER */ + -1, /* (238) expr ::= FLOAT */ + -2, /* (239) expr ::= MINUS FLOAT */ + -2, /* (240) expr ::= PLUS FLOAT */ + -1, /* (241) expr ::= STRING */ + -1, /* (242) expr ::= NOW */ + -1, /* (243) expr ::= VARIABLE */ + -2, /* (244) expr ::= PLUS VARIABLE */ + -2, /* (245) expr ::= MINUS VARIABLE */ + -1, /* (246) expr ::= BOOL */ + -1, /* (247) expr ::= NULL */ + -4, /* (248) expr ::= ID LP exprlist RP */ + -4, /* (249) expr ::= ID LP STAR RP */ + -3, /* (250) expr ::= expr IS NULL */ + -4, /* (251) expr ::= expr IS NOT NULL */ + -3, /* (252) expr ::= expr LT expr */ + -3, /* (253) expr ::= expr GT expr */ + -3, /* (254) expr ::= expr LE expr */ + -3, /* (255) expr ::= expr GE expr */ + -3, /* (256) expr ::= expr NE expr */ + -3, /* (257) expr ::= expr EQ expr */ + -5, /* (258) expr ::= expr BETWEEN expr AND expr */ + -3, /* (259) expr ::= expr AND expr */ + -3, /* (260) expr ::= expr OR expr */ + -3, /* (261) expr ::= expr PLUS expr */ + -3, /* (262) expr ::= expr MINUS expr */ + -3, /* (263) expr ::= expr STAR expr */ + -3, /* (264) expr ::= expr SLASH expr */ + -3, /* (265) expr ::= expr REM expr */ + -3, /* (266) expr ::= expr LIKE expr */ + -5, /* (267) expr ::= expr IN LP exprlist RP */ + -3, /* (268) exprlist ::= exprlist COMMA expritem */ + -1, /* (269) exprlist ::= expritem */ + -1, /* (270) expritem ::= expr */ + 0, /* (271) expritem ::= */ + -3, /* (272) cmd ::= RESET QUERY CACHE */ + -3, /* (273) cmd ::= SYNCDB ids REPLICA */ + -7, /* (274) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (275) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + -7, /* (276) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ + -7, /* (277) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + -7, /* (278) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + -8, /* (279) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + -9, /* (280) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + -7, /* (281) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ + -7, /* (282) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (283) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + -7, /* (284) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ + -7, /* (285) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + -7, /* (286) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + -8, /* (287) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + -9, /* (288) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ + -7, /* (289) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ + -3, /* (290) cmd ::= KILL CONNECTION INTEGER */ + -5, /* (291) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + -5, /* (292) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2139,59 +2649,21 @@ static void yy_accept(yyParser*); /* Forward Declaration */ ** only called from one place, optimizing compilers will in-line it, which ** means that the extra parameters have no performance impact. */ -static void yy_reduce( +static YYACTIONTYPE yy_reduce( yyParser *yypParser, /* The parser */ unsigned int yyruleno, /* Number of the rule by which to reduce */ int yyLookahead, /* Lookahead token, or YYNOCODE if none */ ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + ParseCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ - int yyact; /* The next action */ + YYACTIONTYPE yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - ParseARG_FETCH; + ParseARG_FETCH (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; -#ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; - if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", - yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); - }else{ - fprintf(yyTraceFILE, "%sReduce %d [%s].\n", - yyTracePrompt, yyruleno, yyRuleName[yyruleno]); - } - } -#endif /* NDEBUG */ - - /* Check that the stack is large enough to grow by a single entry - ** if the RHS of the rule is empty. This ensures that there is room - ** enough on the stack to push the LHS value */ - if( yyRuleInfo[yyruleno].nrhs==0 ){ -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - return; - } - yymsp = yypParser->yytos; - } -#endif - } switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -2205,9 +2677,9 @@ static void yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* program ::= cmd */ - case 139: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==139); - case 140: /* cmd ::= CREATE TABLE create_stable_args */ yytestcase(yyruleno==140); - case 141: /* cmd ::= CREATE STABLE create_stable_args */ yytestcase(yyruleno==141); + case 138: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==138); + case 139: /* cmd ::= CREATE TABLE create_stable_args */ yytestcase(yyruleno==139); + case 140: /* cmd ::= CREATE STABLE create_stable_args */ yytestcase(yyruleno==140); {} break; case 1: /* cmd ::= SHOW DATABASES */ @@ -2317,144 +2789,137 @@ static void yy_reduce( setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; - case 29: /* cmd ::= SHOW dbPrefix VGROUPS ids */ -{ - SStrToken token; - tSetDbName(&token, &yymsp[-2].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); -} - break; - case 30: /* cmd ::= DROP TABLE ifexists ids cpxName */ + case 29: /* cmd ::= DROP TABLE ifexists ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, -1); } break; - case 31: /* cmd ::= DROP STABLE ifexists ids cpxName */ + case 30: /* cmd ::= DROP STABLE ifexists ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, TSDB_SUPER_TABLE); } break; - case 32: /* cmd ::= DROP DATABASE ifexists ids */ + case 31: /* cmd ::= DROP DATABASE ifexists ids */ { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_DEFAULT, -1); } break; - case 33: /* cmd ::= DROP TOPIC ifexists ids */ + case 32: /* cmd ::= DROP TOPIC ifexists ids */ { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_TOPIC, -1); } break; - case 34: /* cmd ::= DROP FUNCTION ids */ + case 33: /* cmd ::= DROP FUNCTION ids */ { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &yymsp[0].minor.yy0); } break; - case 35: /* cmd ::= DROP DNODE ids */ + case 34: /* cmd ::= DROP DNODE ids */ { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } break; - case 36: /* cmd ::= DROP USER ids */ + case 35: /* cmd ::= DROP USER ids */ { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &yymsp[0].minor.yy0); } break; - case 37: /* cmd ::= DROP ACCOUNT ids */ + case 36: /* cmd ::= DROP ACCOUNT ids */ { setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &yymsp[0].minor.yy0); } break; - case 38: /* cmd ::= USE ids */ + case 37: /* cmd ::= USE ids */ { setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &yymsp[0].minor.yy0);} break; - case 39: /* cmd ::= DESCRIBE ids cpxName */ - case 40: /* cmd ::= DESC ids cpxName */ yytestcase(yyruleno==40); + case 38: /* cmd ::= DESCRIBE ids cpxName */ + case 39: /* cmd ::= DESC ids cpxName */ yytestcase(yyruleno==39); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &yymsp[-1].minor.yy0); } break; - case 41: /* cmd ::= ALTER USER ids PASS ids */ + case 40: /* cmd ::= ALTER USER ids PASS ids */ { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 42: /* cmd ::= ALTER USER ids PRIVILEGE ids */ + case 41: /* cmd ::= ALTER USER ids PRIVILEGE ids */ { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} break; - case 43: /* cmd ::= ALTER DNODE ids ids */ + case 42: /* cmd ::= ALTER DNODE ids ids */ { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 44: /* cmd ::= ALTER DNODE ids ids ids */ + case 43: /* cmd ::= ALTER DNODE ids ids ids */ { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 45: /* cmd ::= ALTER LOCAL ids */ + case 44: /* cmd ::= ALTER LOCAL ids */ { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &yymsp[0].minor.yy0); } break; - case 46: /* cmd ::= ALTER LOCAL ids ids */ + case 45: /* cmd ::= ALTER LOCAL ids ids */ { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 47: /* cmd ::= ALTER DATABASE ids alter_db_optr */ - case 48: /* cmd ::= ALTER TOPIC ids alter_topic_optr */ yytestcase(yyruleno==48); + case 46: /* cmd ::= ALTER DATABASE ids alter_db_optr */ + case 47: /* cmd ::= ALTER TOPIC ids alter_topic_optr */ yytestcase(yyruleno==47); { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy90, &t);} break; - case 49: /* cmd ::= ALTER ACCOUNT ids acct_optr */ + case 48: /* cmd ::= ALTER ACCOUNT ids acct_optr */ { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy171);} break; - case 50: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + case 49: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy171);} break; - case 51: /* cmd ::= COMPACT VNODES IN LP exprlist RP */ + case 50: /* cmd ::= COMPACT VNODES IN LP exprlist RP */ { setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, yymsp[-1].minor.yy421);} break; - case 52: /* ids ::= ID */ - case 53: /* ids ::= STRING */ yytestcase(yyruleno==53); + case 51: /* ids ::= ID */ + case 52: /* ids ::= STRING */ yytestcase(yyruleno==52); {yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 54: /* ifexists ::= IF EXISTS */ + case 53: /* ifexists ::= IF EXISTS */ { yymsp[-1].minor.yy0.n = 1;} break; - case 55: /* ifexists ::= */ - case 57: /* ifnotexists ::= */ yytestcase(yyruleno==57); - case 181: /* distinct ::= */ yytestcase(yyruleno==181); + case 54: /* ifexists ::= */ + case 56: /* ifnotexists ::= */ yytestcase(yyruleno==56); + case 182: /* distinct ::= */ yytestcase(yyruleno==182); { yymsp[1].minor.yy0.n = 0;} break; - case 56: /* ifnotexists ::= IF NOT EXISTS */ + case 55: /* ifnotexists ::= IF NOT EXISTS */ { yymsp[-2].minor.yy0.n = 1;} break; - case 58: /* cmd ::= CREATE DNODE ids */ + case 57: /* cmd ::= CREATE DNODE ids */ { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; - case 59: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + case 58: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy171);} break; - case 60: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - case 61: /* cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ yytestcase(yyruleno==61); + case 59: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + case 60: /* cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ yytestcase(yyruleno==60); { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy90, &yymsp[-2].minor.yy0);} break; - case 62: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + case 61: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy183, &yymsp[0].minor.yy0, 1);} break; - case 63: /* cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + case 62: /* cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy183, &yymsp[0].minor.yy0, 2);} break; - case 64: /* cmd ::= CREATE USER ids PASS ids */ + case 63: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} break; - case 65: /* bufsize ::= */ - case 67: /* pps ::= */ yytestcase(yyruleno==67); - case 69: /* tseries ::= */ yytestcase(yyruleno==69); - case 71: /* dbs ::= */ yytestcase(yyruleno==71); - case 73: /* streams ::= */ yytestcase(yyruleno==73); - case 75: /* storage ::= */ yytestcase(yyruleno==75); - case 77: /* qtime ::= */ yytestcase(yyruleno==77); - case 79: /* users ::= */ yytestcase(yyruleno==79); - case 81: /* conns ::= */ yytestcase(yyruleno==81); - case 83: /* state ::= */ yytestcase(yyruleno==83); + case 64: /* bufsize ::= */ + case 66: /* pps ::= */ yytestcase(yyruleno==66); + case 68: /* tseries ::= */ yytestcase(yyruleno==68); + case 70: /* dbs ::= */ yytestcase(yyruleno==70); + case 72: /* streams ::= */ yytestcase(yyruleno==72); + case 74: /* storage ::= */ yytestcase(yyruleno==74); + case 76: /* qtime ::= */ yytestcase(yyruleno==76); + case 78: /* users ::= */ yytestcase(yyruleno==78); + case 80: /* conns ::= */ yytestcase(yyruleno==80); + case 82: /* state ::= */ yytestcase(yyruleno==82); { yymsp[1].minor.yy0.n = 0; } break; - case 66: /* bufsize ::= BUFSIZE INTEGER */ - case 68: /* pps ::= PPS INTEGER */ yytestcase(yyruleno==68); - case 70: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==70); - case 72: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==72); - case 74: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==74); - case 76: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==76); - case 78: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==78); - case 80: /* users ::= USERS INTEGER */ yytestcase(yyruleno==80); - case 82: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==82); - case 84: /* state ::= STATE ids */ yytestcase(yyruleno==84); + case 65: /* bufsize ::= BUFSIZE INTEGER */ + case 67: /* pps ::= PPS INTEGER */ yytestcase(yyruleno==67); + case 69: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==69); + case 71: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==71); + case 73: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==73); + case 75: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==75); + case 77: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==77); + case 79: /* users ::= USERS INTEGER */ yytestcase(yyruleno==79); + case 81: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==81); + case 83: /* state ::= STATE ids */ yytestcase(yyruleno==83); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 85: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + case 84: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { yylhsminor.yy171.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; yylhsminor.yy171.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; @@ -2468,135 +2933,135 @@ static void yy_reduce( } yymsp[-8].minor.yy171 = yylhsminor.yy171; break; - case 86: /* intitemlist ::= intitemlist COMMA intitem */ - case 155: /* tagitemlist ::= tagitemlist COMMA tagitem */ yytestcase(yyruleno==155); + case 85: /* intitemlist ::= intitemlist COMMA intitem */ + case 154: /* tagitemlist ::= tagitemlist COMMA tagitem */ yytestcase(yyruleno==154); { yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); } yymsp[-2].minor.yy421 = yylhsminor.yy421; break; - case 87: /* intitemlist ::= intitem */ - case 156: /* tagitemlist ::= tagitem */ yytestcase(yyruleno==156); + case 86: /* intitemlist ::= intitem */ + case 155: /* tagitemlist ::= tagitem */ yytestcase(yyruleno==155); { yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); } yymsp[0].minor.yy421 = yylhsminor.yy421; break; - case 88: /* intitem ::= INTEGER */ - case 157: /* tagitem ::= INTEGER */ yytestcase(yyruleno==157); - case 158: /* tagitem ::= FLOAT */ yytestcase(yyruleno==158); - case 159: /* tagitem ::= STRING */ yytestcase(yyruleno==159); - case 160: /* tagitem ::= BOOL */ yytestcase(yyruleno==160); + case 87: /* intitem ::= INTEGER */ + case 156: /* tagitem ::= INTEGER */ yytestcase(yyruleno==156); + case 157: /* tagitem ::= FLOAT */ yytestcase(yyruleno==157); + case 158: /* tagitem ::= STRING */ yytestcase(yyruleno==158); + case 159: /* tagitem ::= BOOL */ yytestcase(yyruleno==159); { toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } yymsp[0].minor.yy430 = yylhsminor.yy430; break; - case 89: /* keep ::= KEEP intitemlist */ + case 88: /* keep ::= KEEP intitemlist */ { yymsp[-1].minor.yy421 = yymsp[0].minor.yy421; } break; - case 90: /* cache ::= CACHE INTEGER */ - case 91: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==91); - case 92: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==92); - case 93: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==93); - case 94: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==94); - case 95: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==95); - case 96: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==96); - case 97: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==97); - case 98: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==98); - case 99: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==99); - case 100: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==100); - case 101: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==101); - case 102: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==102); - case 103: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==103); - case 104: /* partitions ::= PARTITIONS INTEGER */ yytestcase(yyruleno==104); + case 89: /* cache ::= CACHE INTEGER */ + case 90: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==90); + case 91: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==91); + case 92: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==92); + case 93: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==93); + case 94: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==94); + case 95: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==95); + case 96: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==96); + case 97: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==97); + case 98: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==98); + case 99: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==99); + case 100: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==100); + case 101: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==101); + case 102: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==102); + case 103: /* partitions ::= PARTITIONS INTEGER */ yytestcase(yyruleno==103); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 105: /* db_optr ::= */ + case 104: /* db_optr ::= */ {setDefaultCreateDbOption(&yymsp[1].minor.yy90); yymsp[1].minor.yy90.dbType = TSDB_DB_TYPE_DEFAULT;} break; - case 106: /* db_optr ::= db_optr cache */ + case 105: /* db_optr ::= db_optr cache */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 107: /* db_optr ::= db_optr replica */ - case 124: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==124); + case 106: /* db_optr ::= db_optr replica */ + case 123: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==123); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 108: /* db_optr ::= db_optr quorum */ - case 125: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==125); + case 107: /* db_optr ::= db_optr quorum */ + case 124: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==124); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 109: /* db_optr ::= db_optr days */ + case 108: /* db_optr ::= db_optr days */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 110: /* db_optr ::= db_optr minrows */ + case 109: /* db_optr ::= db_optr minrows */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 111: /* db_optr ::= db_optr maxrows */ + case 110: /* db_optr ::= db_optr maxrows */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 112: /* db_optr ::= db_optr blocks */ - case 127: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==127); + case 111: /* db_optr ::= db_optr blocks */ + case 126: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==126); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 113: /* db_optr ::= db_optr ctime */ + case 112: /* db_optr ::= db_optr ctime */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 114: /* db_optr ::= db_optr wal */ + case 113: /* db_optr ::= db_optr wal */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 115: /* db_optr ::= db_optr fsync */ + case 114: /* db_optr ::= db_optr fsync */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 116: /* db_optr ::= db_optr comp */ - case 128: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==128); + case 115: /* db_optr ::= db_optr comp */ + case 127: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==127); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 117: /* db_optr ::= db_optr prec */ + case 116: /* db_optr ::= db_optr prec */ { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.precision = yymsp[0].minor.yy0; } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 118: /* db_optr ::= db_optr keep */ - case 126: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==126); + case 117: /* db_optr ::= db_optr keep */ + case 125: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==125); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.keep = yymsp[0].minor.yy421; } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 119: /* db_optr ::= db_optr update */ - case 129: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==129); + case 118: /* db_optr ::= db_optr update */ + case 128: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==128); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 120: /* db_optr ::= db_optr cachelast */ - case 130: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==130); + case 119: /* db_optr ::= db_optr cachelast */ + case 129: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==129); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 121: /* topic_optr ::= db_optr */ - case 131: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==131); + case 120: /* topic_optr ::= db_optr */ + case 130: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==130); { yylhsminor.yy90 = yymsp[0].minor.yy90; yylhsminor.yy90.dbType = TSDB_DB_TYPE_TOPIC; } yymsp[0].minor.yy90 = yylhsminor.yy90; break; - case 122: /* topic_optr ::= topic_optr partitions */ - case 132: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==132); + case 121: /* topic_optr ::= topic_optr partitions */ + case 131: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==131); { yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy90 = yylhsminor.yy90; break; - case 123: /* alter_db_optr ::= */ + case 122: /* alter_db_optr ::= */ { setDefaultCreateDbOption(&yymsp[1].minor.yy90); yymsp[1].minor.yy90.dbType = TSDB_DB_TYPE_DEFAULT;} break; - case 133: /* typename ::= ids */ + case 132: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; tSetColumnType (&yylhsminor.yy183, &yymsp[0].minor.yy0); } yymsp[0].minor.yy183 = yylhsminor.yy183; break; - case 134: /* typename ::= ids LP signed RP */ + case 133: /* typename ::= ids LP signed RP */ { if (yymsp[-1].minor.yy325 <= 0) { yymsp[-3].minor.yy0.type = 0; @@ -2608,7 +3073,7 @@ static void yy_reduce( } yymsp[-3].minor.yy183 = yylhsminor.yy183; break; - case 135: /* typename ::= ids UNSIGNED */ + case 134: /* typename ::= ids UNSIGNED */ { yymsp[-1].minor.yy0.type = 0; yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); @@ -2616,20 +3081,20 @@ static void yy_reduce( } yymsp[-1].minor.yy183 = yylhsminor.yy183; break; - case 136: /* signed ::= INTEGER */ + case 135: /* signed ::= INTEGER */ { yylhsminor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[0].minor.yy325 = yylhsminor.yy325; break; - case 137: /* signed ::= PLUS INTEGER */ + case 136: /* signed ::= PLUS INTEGER */ { yymsp[-1].minor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 138: /* signed ::= MINUS INTEGER */ + case 137: /* signed ::= MINUS INTEGER */ { yymsp[-1].minor.yy325 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 142: /* cmd ::= CREATE TABLE create_table_list */ + case 141: /* cmd ::= CREATE TABLE create_table_list */ { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy438;} break; - case 143: /* create_table_list ::= create_from_stable */ + case 142: /* create_table_list ::= create_from_stable */ { SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); @@ -2640,14 +3105,14 @@ static void yy_reduce( } yymsp[0].minor.yy438 = yylhsminor.yy438; break; - case 144: /* create_table_list ::= create_table_list create_from_stable */ + case 143: /* create_table_list ::= create_table_list create_from_stable */ { taosArrayPush(yymsp[-1].minor.yy438->childTableInfo, &yymsp[0].minor.yy152); yylhsminor.yy438 = yymsp[-1].minor.yy438; } yymsp[-1].minor.yy438 = yylhsminor.yy438; break; - case 145: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + case 144: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { yylhsminor.yy438 = tSetCreateTableInfo(yymsp[-1].minor.yy421, NULL, NULL, TSQL_CREATE_TABLE); setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); @@ -2657,7 +3122,7 @@ static void yy_reduce( } yymsp[-5].minor.yy438 = yylhsminor.yy438; break; - case 146: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + case 145: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { yylhsminor.yy438 = tSetCreateTableInfo(yymsp[-5].minor.yy421, yymsp[-1].minor.yy421, NULL, TSQL_CREATE_STABLE); setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); @@ -2667,7 +3132,7 @@ static void yy_reduce( } yymsp[-9].minor.yy438 = yylhsminor.yy438; break; - case 147: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + case 146: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; @@ -2675,7 +3140,7 @@ static void yy_reduce( } yymsp[-9].minor.yy152 = yylhsminor.yy152; break; - case 148: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ + case 147: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ { yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; @@ -2683,15 +3148,15 @@ static void yy_reduce( } yymsp[-12].minor.yy152 = yylhsminor.yy152; break; - case 149: /* tagNamelist ::= tagNamelist COMMA ids */ + case 148: /* tagNamelist ::= tagNamelist COMMA ids */ {taosArrayPush(yymsp[-2].minor.yy421, &yymsp[0].minor.yy0); yylhsminor.yy421 = yymsp[-2].minor.yy421; } yymsp[-2].minor.yy421 = yylhsminor.yy421; break; - case 150: /* tagNamelist ::= ids */ + case 149: /* tagNamelist ::= ids */ {yylhsminor.yy421 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy421, &yymsp[0].minor.yy0);} yymsp[0].minor.yy421 = yylhsminor.yy421; break; - case 151: /* create_table_args ::= ifnotexists ids cpxName AS select */ + case 150: /* create_table_args ::= ifnotexists ids cpxName AS select */ { yylhsminor.yy438 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy56, TSQL_CREATE_STREAM); setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); @@ -2701,32 +3166,44 @@ static void yy_reduce( } yymsp[-4].minor.yy438 = yylhsminor.yy438; break; - case 152: /* columnlist ::= columnlist COMMA column */ + case 151: /* columnlist ::= columnlist COMMA column */ {taosArrayPush(yymsp[-2].minor.yy421, &yymsp[0].minor.yy183); yylhsminor.yy421 = yymsp[-2].minor.yy421; } yymsp[-2].minor.yy421 = yylhsminor.yy421; break; - case 153: /* columnlist ::= column */ + case 152: /* columnlist ::= column */ {yylhsminor.yy421 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy421, &yymsp[0].minor.yy183);} yymsp[0].minor.yy421 = yylhsminor.yy421; break; - case 154: /* column ::= ids typename */ + case 153: /* column ::= ids typename */ { tSetColumnInfo(&yylhsminor.yy183, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy183); } yymsp[-1].minor.yy183 = yylhsminor.yy183; break; - case 161: /* tagitem ::= NULL */ + case 160: /* tagitem ::= NULL */ { yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } yymsp[0].minor.yy430 = yylhsminor.yy430; break; - case 162: /* tagitem ::= NOW */ -{ yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0);} + case 161: /* tagitem ::= NOW */ +{ yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreateExt(&yylhsminor.yy430, &yymsp[0].minor.yy0, TK_NOW);} yymsp[0].minor.yy430 = yylhsminor.yy430; break; - case 163: /* tagitem ::= MINUS INTEGER */ - case 164: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==164); - case 165: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==165); - case 166: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==166); + case 162: /* tagitem ::= NOW PLUS VARIABLE */ +{ + yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; + tVariantCreateExt(&yymsp[-2].minor.yy430, &yymsp[0].minor.yy0, TK_PLUS); +} + break; + case 163: /* tagitem ::= NOW MINUS VARIABLE */ +{ + yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; + tVariantCreateExt(&yymsp[-2].minor.yy430, &yymsp[0].minor.yy0, TK_MINUS); +} + break; + case 164: /* tagitem ::= MINUS INTEGER */ + case 165: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==165); + case 166: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==166); + case 167: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==167); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; @@ -2735,150 +3212,150 @@ static void yy_reduce( } yymsp[-1].minor.yy430 = yylhsminor.yy430; break; - case 167: /* select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ + case 168: /* select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ { yylhsminor.yy56 = tSetQuerySqlNode(&yymsp[-13].minor.yy0, yymsp[-12].minor.yy421, yymsp[-11].minor.yy8, yymsp[-10].minor.yy439, yymsp[-4].minor.yy421, yymsp[-2].minor.yy421, &yymsp[-9].minor.yy400, &yymsp[-7].minor.yy147, &yymsp[-6].minor.yy40, &yymsp[-8].minor.yy0, yymsp[-5].minor.yy421, &yymsp[0].minor.yy166, &yymsp[-1].minor.yy166, yymsp[-3].minor.yy439); } yymsp[-13].minor.yy56 = yylhsminor.yy56; break; - case 168: /* select ::= LP select RP */ + case 169: /* select ::= LP select RP */ {yymsp[-2].minor.yy56 = yymsp[-1].minor.yy56;} break; - case 169: /* union ::= select */ + case 170: /* union ::= select */ { yylhsminor.yy421 = setSubclause(NULL, yymsp[0].minor.yy56); } yymsp[0].minor.yy421 = yylhsminor.yy421; break; - case 170: /* union ::= union UNION ALL select */ + case 171: /* union ::= union UNION ALL select */ { yylhsminor.yy421 = appendSelectClause(yymsp[-3].minor.yy421, yymsp[0].minor.yy56); } yymsp[-3].minor.yy421 = yylhsminor.yy421; break; - case 171: /* cmd ::= union */ + case 172: /* cmd ::= union */ { setSqlInfo(pInfo, yymsp[0].minor.yy421, NULL, TSDB_SQL_SELECT); } break; - case 172: /* select ::= SELECT selcollist */ + case 173: /* select ::= SELECT selcollist */ { yylhsminor.yy56 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy421, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } yymsp[-1].minor.yy56 = yylhsminor.yy56; break; - case 173: /* sclp ::= selcollist COMMA */ + case 174: /* sclp ::= selcollist COMMA */ {yylhsminor.yy421 = yymsp[-1].minor.yy421;} yymsp[-1].minor.yy421 = yylhsminor.yy421; break; - case 174: /* sclp ::= */ - case 206: /* orderby_opt ::= */ yytestcase(yyruleno==206); + case 175: /* sclp ::= */ + case 207: /* orderby_opt ::= */ yytestcase(yyruleno==207); {yymsp[1].minor.yy421 = 0;} break; - case 175: /* selcollist ::= sclp distinct expr as */ + case 176: /* selcollist ::= sclp distinct expr as */ { yylhsminor.yy421 = tSqlExprListAppend(yymsp[-3].minor.yy421, yymsp[-1].minor.yy439, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } yymsp[-3].minor.yy421 = yylhsminor.yy421; break; - case 176: /* selcollist ::= sclp STAR */ + case 177: /* selcollist ::= sclp STAR */ { - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); + tSqlExpr *pNode = tSqlExprCreateIdValue(pInfo, NULL, TK_ALL); yylhsminor.yy421 = tSqlExprListAppend(yymsp[-1].minor.yy421, pNode, 0, 0); } yymsp[-1].minor.yy421 = yylhsminor.yy421; break; - case 177: /* as ::= AS ids */ + case 178: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 178: /* as ::= ids */ + case 179: /* as ::= ids */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 179: /* as ::= */ + case 180: /* as ::= */ { yymsp[1].minor.yy0.n = 0; } break; - case 180: /* distinct ::= DISTINCT */ + case 181: /* distinct ::= DISTINCT */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 182: /* from ::= FROM tablelist */ - case 183: /* from ::= FROM sub */ yytestcase(yyruleno==183); + case 183: /* from ::= FROM tablelist */ + case 184: /* from ::= FROM sub */ yytestcase(yyruleno==184); {yymsp[-1].minor.yy8 = yymsp[0].minor.yy8;} break; - case 184: /* sub ::= LP union RP */ + case 185: /* sub ::= LP union RP */ {yymsp[-2].minor.yy8 = addSubqueryElem(NULL, yymsp[-1].minor.yy421, NULL);} break; - case 185: /* sub ::= LP union RP ids */ + case 186: /* sub ::= LP union RP ids */ {yymsp[-3].minor.yy8 = addSubqueryElem(NULL, yymsp[-2].minor.yy421, &yymsp[0].minor.yy0);} break; - case 186: /* sub ::= sub COMMA LP union RP ids */ + case 187: /* sub ::= sub COMMA LP union RP ids */ {yylhsminor.yy8 = addSubqueryElem(yymsp[-5].minor.yy8, yymsp[-2].minor.yy421, &yymsp[0].minor.yy0);} yymsp[-5].minor.yy8 = yylhsminor.yy8; break; - case 187: /* tablelist ::= ids cpxName */ + case 188: /* tablelist ::= ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy8 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); } yymsp[-1].minor.yy8 = yylhsminor.yy8; break; - case 188: /* tablelist ::= ids cpxName ids */ + case 189: /* tablelist ::= ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; yylhsminor.yy8 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy8 = yylhsminor.yy8; break; - case 189: /* tablelist ::= tablelist COMMA ids cpxName */ + case 190: /* tablelist ::= tablelist COMMA ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy8 = setTableNameList(yymsp[-3].minor.yy8, &yymsp[-1].minor.yy0, NULL); } yymsp[-3].minor.yy8 = yylhsminor.yy8; break; - case 190: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 191: /* tablelist ::= tablelist COMMA ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; yylhsminor.yy8 = setTableNameList(yymsp[-4].minor.yy8, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } yymsp[-4].minor.yy8 = yylhsminor.yy8; break; - case 191: /* tmvar ::= VARIABLE */ + case 192: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 192: /* interval_option ::= intervalKey LP tmvar RP */ + case 193: /* interval_option ::= intervalKey LP tmvar RP */ {yylhsminor.yy400.interval = yymsp[-1].minor.yy0; yylhsminor.yy400.offset.n = 0; yylhsminor.yy400.token = yymsp[-3].minor.yy104;} yymsp[-3].minor.yy400 = yylhsminor.yy400; break; - case 193: /* interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ + case 194: /* interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ {yylhsminor.yy400.interval = yymsp[-3].minor.yy0; yylhsminor.yy400.offset = yymsp[-1].minor.yy0; yylhsminor.yy400.token = yymsp[-5].minor.yy104;} yymsp[-5].minor.yy400 = yylhsminor.yy400; break; - case 194: /* interval_option ::= */ + case 195: /* interval_option ::= */ {memset(&yymsp[1].minor.yy400, 0, sizeof(yymsp[1].minor.yy400));} break; - case 195: /* intervalKey ::= INTERVAL */ + case 196: /* intervalKey ::= INTERVAL */ {yymsp[0].minor.yy104 = TK_INTERVAL;} break; - case 196: /* intervalKey ::= EVERY */ + case 197: /* intervalKey ::= EVERY */ {yymsp[0].minor.yy104 = TK_EVERY; } break; - case 197: /* session_option ::= */ + case 198: /* session_option ::= */ {yymsp[1].minor.yy147.col.n = 0; yymsp[1].minor.yy147.gap.n = 0;} break; - case 198: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + case 199: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; yymsp[-6].minor.yy147.col = yymsp[-4].minor.yy0; yymsp[-6].minor.yy147.gap = yymsp[-1].minor.yy0; } break; - case 199: /* windowstate_option ::= */ + case 200: /* windowstate_option ::= */ { yymsp[1].minor.yy40.col.n = 0; yymsp[1].minor.yy40.col.z = NULL;} break; - case 200: /* windowstate_option ::= STATE_WINDOW LP ids RP */ + case 201: /* windowstate_option ::= STATE_WINDOW LP ids RP */ { yymsp[-3].minor.yy40.col = yymsp[-1].minor.yy0; } break; - case 201: /* fill_opt ::= */ + case 202: /* fill_opt ::= */ { yymsp[1].minor.yy421 = 0; } break; - case 202: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 203: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); @@ -2888,34 +3365,34 @@ static void yy_reduce( yymsp[-5].minor.yy421 = yymsp[-1].minor.yy421; } break; - case 203: /* fill_opt ::= FILL LP ID RP */ + case 204: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-3].minor.yy421 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 204: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 205: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 205: /* sliding_opt ::= */ + case 206: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 207: /* orderby_opt ::= ORDER BY sortlist */ + case 208: /* orderby_opt ::= ORDER BY sortlist */ {yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} break; - case 208: /* sortlist ::= sortlist COMMA item sortorder */ + case 209: /* sortlist ::= sortlist COMMA item sortorder */ { yylhsminor.yy421 = tVariantListAppend(yymsp[-3].minor.yy421, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); } yymsp[-3].minor.yy421 = yylhsminor.yy421; break; - case 209: /* sortlist ::= item sortorder */ + case 210: /* sortlist ::= item sortorder */ { yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); } yymsp[-1].minor.yy421 = yylhsminor.yy421; break; - case 210: /* item ::= ids cpxName */ + case 211: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; @@ -2924,227 +3401,227 @@ static void yy_reduce( } yymsp[-1].minor.yy430 = yylhsminor.yy430; break; - case 211: /* sortorder ::= ASC */ + case 212: /* sortorder ::= ASC */ { yymsp[0].minor.yy96 = TSDB_ORDER_ASC; } break; - case 212: /* sortorder ::= DESC */ + case 213: /* sortorder ::= DESC */ { yymsp[0].minor.yy96 = TSDB_ORDER_DESC;} break; - case 213: /* sortorder ::= */ + case 214: /* sortorder ::= */ { yymsp[1].minor.yy96 = TSDB_ORDER_ASC; } break; - case 214: /* groupby_opt ::= */ + case 215: /* groupby_opt ::= */ { yymsp[1].minor.yy421 = 0;} break; - case 215: /* groupby_opt ::= GROUP BY grouplist */ + case 216: /* groupby_opt ::= GROUP BY grouplist */ { yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} break; - case 216: /* grouplist ::= grouplist COMMA item */ + case 217: /* grouplist ::= grouplist COMMA item */ { yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); } yymsp[-2].minor.yy421 = yylhsminor.yy421; break; - case 217: /* grouplist ::= item */ + case 218: /* grouplist ::= item */ { yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); } yymsp[0].minor.yy421 = yylhsminor.yy421; break; - case 218: /* having_opt ::= */ - case 228: /* where_opt ::= */ yytestcase(yyruleno==228); - case 270: /* expritem ::= */ yytestcase(yyruleno==270); + case 219: /* having_opt ::= */ + case 229: /* where_opt ::= */ yytestcase(yyruleno==229); + case 271: /* expritem ::= */ yytestcase(yyruleno==271); {yymsp[1].minor.yy439 = 0;} break; - case 219: /* having_opt ::= HAVING expr */ - case 229: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==229); + case 220: /* having_opt ::= HAVING expr */ + case 230: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==230); {yymsp[-1].minor.yy439 = yymsp[0].minor.yy439;} break; - case 220: /* limit_opt ::= */ - case 224: /* slimit_opt ::= */ yytestcase(yyruleno==224); + case 221: /* limit_opt ::= */ + case 225: /* slimit_opt ::= */ yytestcase(yyruleno==225); {yymsp[1].minor.yy166.limit = -1; yymsp[1].minor.yy166.offset = 0;} break; - case 221: /* limit_opt ::= LIMIT signed */ - case 225: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==225); + case 222: /* limit_opt ::= LIMIT signed */ + case 226: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==226); {yymsp[-1].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-1].minor.yy166.offset = 0;} break; - case 222: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 223: /* limit_opt ::= LIMIT signed OFFSET signed */ { yymsp[-3].minor.yy166.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[0].minor.yy325;} break; - case 223: /* limit_opt ::= LIMIT signed COMMA signed */ + case 224: /* limit_opt ::= LIMIT signed COMMA signed */ { yymsp[-3].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[-2].minor.yy325;} break; - case 226: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ + case 227: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ {yymsp[-3].minor.yy166.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[0].minor.yy325;} break; - case 227: /* slimit_opt ::= SLIMIT signed COMMA signed */ + case 228: /* slimit_opt ::= SLIMIT signed COMMA signed */ {yymsp[-3].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[-2].minor.yy325;} break; - case 230: /* expr ::= LP expr RP */ + case 231: /* expr ::= LP expr RP */ {yylhsminor.yy439 = yymsp[-1].minor.yy439; yylhsminor.yy439->exprToken.z = yymsp[-2].minor.yy0.z; yylhsminor.yy439->exprToken.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 231: /* expr ::= ID */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} + case 232: /* expr ::= ID */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_ID);} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 232: /* expr ::= ID DOT ID */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} + case 233: /* expr ::= ID DOT ID */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[-2].minor.yy0, TK_ID);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 233: /* expr ::= ID DOT STAR */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} + case 234: /* expr ::= ID DOT STAR */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[-2].minor.yy0, TK_ALL);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 234: /* expr ::= INTEGER */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} + case 235: /* expr ::= INTEGER */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_INTEGER);} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 235: /* expr ::= MINUS INTEGER */ - case 236: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==236); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} + case 236: /* expr ::= MINUS INTEGER */ + case 237: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==237); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[-1].minor.yy0, TK_INTEGER);} yymsp[-1].minor.yy439 = yylhsminor.yy439; break; - case 237: /* expr ::= FLOAT */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} + case 238: /* expr ::= FLOAT */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_FLOAT);} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 238: /* expr ::= MINUS FLOAT */ - case 239: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==239); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} + case 239: /* expr ::= MINUS FLOAT */ + case 240: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==240); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[-1].minor.yy0, TK_FLOAT);} yymsp[-1].minor.yy439 = yylhsminor.yy439; break; - case 240: /* expr ::= STRING */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} + case 241: /* expr ::= STRING */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_STRING);} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 241: /* expr ::= NOW */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } + case 242: /* expr ::= NOW */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_NOW); } yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 242: /* expr ::= VARIABLE */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} + case 243: /* expr ::= VARIABLE */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_VARIABLE);} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 243: /* expr ::= PLUS VARIABLE */ - case 244: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==244); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} + case 244: /* expr ::= PLUS VARIABLE */ + case 245: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==245); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[-1].minor.yy0, TK_VARIABLE);} yymsp[-1].minor.yy439 = yylhsminor.yy439; break; - case 245: /* expr ::= BOOL */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} + case 246: /* expr ::= BOOL */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_BOOL);} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 246: /* expr ::= NULL */ -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} + case 247: /* expr ::= NULL */ +{ yylhsminor.yy439 = tSqlExprCreateIdValue(pInfo, &yymsp[0].minor.yy0, TK_NULL);} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 247: /* expr ::= ID LP exprlist RP */ + case 248: /* expr ::= ID LP exprlist RP */ { tStrTokenAppend(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy439 = tSqlExprCreateFunction(yymsp[-1].minor.yy421, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy439 = yylhsminor.yy439; break; - case 248: /* expr ::= ID LP STAR RP */ + case 249: /* expr ::= ID LP STAR RP */ { tStrTokenAppend(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy439 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy439 = yylhsminor.yy439; break; - case 249: /* expr ::= expr IS NULL */ + case 250: /* expr ::= expr IS NULL */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, NULL, TK_ISNULL);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 250: /* expr ::= expr IS NOT NULL */ + case 251: /* expr ::= expr IS NOT NULL */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-3].minor.yy439, NULL, TK_NOTNULL);} yymsp[-3].minor.yy439 = yylhsminor.yy439; break; - case 251: /* expr ::= expr LT expr */ + case 252: /* expr ::= expr LT expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LT);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 252: /* expr ::= expr GT expr */ + case 253: /* expr ::= expr GT expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_GT);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 253: /* expr ::= expr LE expr */ + case 254: /* expr ::= expr LE expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LE);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 254: /* expr ::= expr GE expr */ + case 255: /* expr ::= expr GE expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_GE);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 255: /* expr ::= expr NE expr */ + case 256: /* expr ::= expr NE expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_NE);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 256: /* expr ::= expr EQ expr */ + case 257: /* expr ::= expr EQ expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_EQ);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 257: /* expr ::= expr BETWEEN expr AND expr */ + case 258: /* expr ::= expr BETWEEN expr AND expr */ { tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy439); yylhsminor.yy439 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy439, yymsp[-2].minor.yy439, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy439, TK_LE), TK_AND);} yymsp[-4].minor.yy439 = yylhsminor.yy439; break; - case 258: /* expr ::= expr AND expr */ + case 259: /* expr ::= expr AND expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_AND);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 259: /* expr ::= expr OR expr */ + case 260: /* expr ::= expr OR expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_OR); } yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 260: /* expr ::= expr PLUS expr */ + case 261: /* expr ::= expr PLUS expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_PLUS); } yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 261: /* expr ::= expr MINUS expr */ + case 262: /* expr ::= expr MINUS expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_MINUS); } yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 262: /* expr ::= expr STAR expr */ + case 263: /* expr ::= expr STAR expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_STAR); } yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 263: /* expr ::= expr SLASH expr */ + case 264: /* expr ::= expr SLASH expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_DIVIDE);} yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 264: /* expr ::= expr REM expr */ + case 265: /* expr ::= expr REM expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_REM); } yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 265: /* expr ::= expr LIKE expr */ + case 266: /* expr ::= expr LIKE expr */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LIKE); } yymsp[-2].minor.yy439 = yylhsminor.yy439; break; - case 266: /* expr ::= expr IN LP exprlist RP */ + case 267: /* expr ::= expr IN LP exprlist RP */ {yylhsminor.yy439 = tSqlExprCreate(yymsp[-4].minor.yy439, (tSqlExpr*)yymsp[-1].minor.yy421, TK_IN); } yymsp[-4].minor.yy439 = yylhsminor.yy439; break; - case 267: /* exprlist ::= exprlist COMMA expritem */ + case 268: /* exprlist ::= exprlist COMMA expritem */ {yylhsminor.yy421 = tSqlExprListAppend(yymsp[-2].minor.yy421,yymsp[0].minor.yy439,0, 0);} yymsp[-2].minor.yy421 = yylhsminor.yy421; break; - case 268: /* exprlist ::= expritem */ + case 269: /* exprlist ::= expritem */ {yylhsminor.yy421 = tSqlExprListAppend(0,yymsp[0].minor.yy439,0, 0);} yymsp[0].minor.yy421 = yylhsminor.yy421; break; - case 269: /* expritem ::= expr */ + case 270: /* expritem ::= expr */ {yylhsminor.yy439 = yymsp[0].minor.yy439;} yymsp[0].minor.yy439 = yylhsminor.yy439; break; - case 271: /* cmd ::= RESET QUERY CACHE */ + case 272: /* cmd ::= RESET QUERY CACHE */ { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 272: /* cmd ::= SYNCDB ids REPLICA */ + case 273: /* cmd ::= SYNCDB ids REPLICA */ { setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &yymsp[-1].minor.yy0);} break; - case 273: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 274: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 274: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 275: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3155,21 +3632,21 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 275: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ + case 276: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 276: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 277: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 277: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 278: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3180,7 +3657,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 278: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 279: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3194,7 +3671,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 279: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 280: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -3206,21 +3683,21 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 280: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ + case 281: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 281: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + case 282: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 282: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + case 283: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3231,21 +3708,21 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 283: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ + case 284: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 284: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + case 285: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 285: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + case 286: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3256,7 +3733,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 286: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + case 287: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3270,7 +3747,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 287: /* cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ + case 288: /* cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -3282,29 +3759,29 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 288: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ + case 289: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 289: /* cmd ::= KILL CONNECTION INTEGER */ + case 290: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 290: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 291: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 291: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 292: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: break; /********** End reduce actions ************************************************/ }; - assert( yyrulenostateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; } /* @@ -3328,7 +3806,8 @@ static void yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); @@ -3339,7 +3818,8 @@ static void yy_parse_failed( ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } #endif /* YYNOERRORRECOVERY */ @@ -3351,7 +3831,8 @@ static void yy_syntax_error( int yymajor, /* The major type of the error token */ ParseTOKENTYPE yyminor /* The minor type of the error token */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ @@ -3377,7 +3858,8 @@ static void yy_syntax_error( assert(len <= outputBufLen); /************ End %syntax_error code ******************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* @@ -3386,7 +3868,8 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); @@ -3401,7 +3884,8 @@ static void yy_accept( /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* The main parser program. @@ -3430,45 +3914,91 @@ void Parse( ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - unsigned int yyact; /* The parser action. */ + YYACTIONTYPE yyact; /* The parser action. */ #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ #endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif - yyParser *yypParser; /* The parser */ + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + ParseCTX_FETCH + ParseARG_STORE - yypParser = (yyParser*)yyp; assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif - ParseARG_STORE; + yyact = yypParser->yytos->stateno; #ifndef NDEBUG if( yyTraceFILE ){ - int stateno = yypParser->yytos->stateno; - if( stateno < YY_MIN_REDUCE ){ + if( yyact < YY_MIN_REDUCE ){ fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],stateno); + yyTracePrompt,yyTokenName[yymajor],yyact); }else{ fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE); + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); } } #endif - do{ - yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + while(1){ /* Exit by "break" */ + assert( yypParser->yytos>=yypParser->yystack ); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); if( yyact >= YY_MIN_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor); + unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */ + assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); +#ifndef NDEBUG + if( yyTraceFILE ){ + int yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == + (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + break; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + break; + } + } +#endif + } + yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor ParseCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,yymajor,yyminor); + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif - yymajor = YYNOCODE; + break; }else if( yyact==YY_ACCEPT_ACTION ){ yypParser->yytos--; yy_accept(yypParser); @@ -3519,10 +4049,9 @@ void Parse( yymajor = YYNOCODE; }else{ while( yypParser->yytos >= yypParser->yystack - && yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yytos->stateno, - YYERRORSYMBOL)) >= YY_MIN_REDUCE + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE ){ yy_pop_parser_stack(yypParser); } @@ -3539,6 +4068,8 @@ void Parse( } yypParser->yyerrcnt = 3; yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; #elif defined(YYNOERRORRECOVERY) /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -3549,8 +4080,7 @@ void Parse( */ yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - + break; #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -3572,10 +4102,10 @@ void Parse( yypParser->yyerrcnt = -1; #endif } - yymajor = YYNOCODE; + break; #endif } - }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack ); + } #ifndef NDEBUG if( yyTraceFILE ){ yyStackEntry *i; @@ -3590,3 +4120,17 @@ void Parse( #endif return; } + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int ParseFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 60a12c26b78626ed81cbc182d76c836c6ee74498..9784a8f354424974f04227f4137569132c6cc72c 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -49,7 +49,7 @@ static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType static void rpcLockCache(int64_t *lockedBy); static void rpcUnlockCache(int64_t *lockedBy); static void rpcCleanConnCache(void *handle, void *tmrId); -static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time); +static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t rpc_time); void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer) { SConnHash **connHashList; @@ -118,7 +118,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in SConnHash * pNode; SConnCache *pCache; - uint64_t time = taosGetTimestampMs(); + uint64_t rpc_time = taosGetTimestampMs(); pCache = (SConnCache *)handle; assert(pCache); @@ -131,7 +131,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pNode->connType = connType; pNode->data = data; pNode->prev = NULL; - pNode->time = time; + pNode->time = rpc_time; rpcLockCache(pCache->lockedBy+hash); @@ -140,7 +140,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pCache->connHashList[hash] = pNode; pCache->count[hash]++; - rpcRemoveExpiredNodes(pCache, pNode->next, hash, time); + rpcRemoveExpiredNodes(pCache, pNode->next, hash, rpc_time); rpcUnlockCache(pCache->lockedBy+hash); @@ -159,15 +159,15 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy pCache = (SConnCache *)handle; assert(pCache); - uint64_t time = taosGetTimestampMs(); + uint64_t rpc_time = taosGetTimestampMs(); hash = rpcHashConn(pCache, fqdn, port, connType); rpcLockCache(pCache->lockedBy+hash); pNode = pCache->connHashList[hash]; while (pNode) { - if (time >= pCache->keepTimer + pNode->time) { - rpcRemoveExpiredNodes(pCache, pNode, hash, time); + if (rpc_time >= pCache->keepTimer + pNode->time) { + rpcRemoveExpiredNodes(pCache, pNode, hash, rpc_time); pNode = NULL; break; } @@ -178,7 +178,7 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy } if (pNode) { - rpcRemoveExpiredNodes(pCache, pNode->next, hash, time); + rpcRemoveExpiredNodes(pCache, pNode->next, hash, rpc_time); if (pNode->prev) { pNode->prev->next = pNode->next; @@ -217,12 +217,12 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { if (pCache->pTimer != tmrId) return; pthread_mutex_lock(&pCache->mutex); - uint64_t time = taosGetTimestampMs(); + uint64_t rpc_time = taosGetTimestampMs(); for (hash = 0; hash < pCache->maxSessions; ++hash) { rpcLockCache(pCache->lockedBy+hash); pNode = pCache->connHashList[hash]; - rpcRemoveExpiredNodes(pCache, pNode, hash, time); + rpcRemoveExpiredNodes(pCache, pNode, hash, rpc_time); rpcUnlockCache(pCache->lockedBy+hash); } @@ -231,8 +231,8 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { pthread_mutex_unlock(&pCache->mutex); } -static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { - if (pNode == NULL || (time < pCache->keepTimer + pNode->time) ) return; +static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t rpc_time) { + if (pNode == NULL || (rpc_time < pCache->keepTimer + pNode->time) ) return; SConnHash *pPrev = pNode->prev, *pNext; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 25495182498f7c1a82f9f9459290e44f082f5eb2..6eb210ee2bbaec396ef6293a65867d2534f25925 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -392,9 +392,9 @@ void taosCleanUpTcpClient(void *chandle) { void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SClientObj * pClientObj = shandle; - int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; - atomic_store_32(&pClientObj->index, index + 1); - SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; + int32_t rpc_index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; + atomic_store_32(&pClientObj->index, rpc_index + 1); + SThreadObj *pThreadObj = pClientObj->pThreadObj[rpc_index]; SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -403,12 +403,12 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin if (fd <= 0) return NULL; #endif - struct sockaddr_in sin; + struct sockaddr_in rpc_sin; uint16_t localPort = 0; - unsigned int addrlen = sizeof(sin); - if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { - localPort = (uint16_t)ntohs(sin.sin_port); + unsigned int addrlen = sizeof(rpc_sin); + if (getsockname(fd, (struct sockaddr *)&rpc_sin, &addrlen) == 0 && + rpc_sin.sin_family == AF_INET && addrlen == sizeof(rpc_sin)) { + localPort = (uint16_t)ntohs(rpc_sin.sin_port); } SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); @@ -488,6 +488,12 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); int32_t size = msgLen + tsRpcOverhead; + // TODO: reason not found yet, workaround to avoid first + if (msgLen <= 0 || size < 0) { + tError("%s %p invalid size for malloc, msgLen:%d, size:%d", pThreadObj->label, pFdObj->thandle, msgLen, size); + return -1; + } + buffer = malloc(size); if (NULL == buffer) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 086a390cb8ea2a95f576cb1bff81dfc79769863a..a908059d5e94d1f3b9c0b9fdf7c5961b81e2c303 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -97,11 +97,11 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads break; } - struct sockaddr_in sin; - unsigned int addrlen = sizeof(sin); - if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { - pConn->localPort = (uint16_t)ntohs(sin.sin_port); + struct sockaddr_in rpc_sin; + unsigned int addrlen = sizeof(rpc_sin); + if (getsockname(pConn->fd, (struct sockaddr *)&rpc_sin, &addrlen) == 0 && + rpc_sin.sin_family == AF_INET && addrlen == sizeof(rpc_sin)) { + pConn->localPort = (uint16_t)ntohs(rpc_sin.sin_port); } tstrncpy(pConn->label, label, sizeof(pConn->label)); diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 2f4433f1bb32e965de66a40d7d6ae36c6804a06c..1ef0b33f52dbe9b80431ca5c015cccd3ed56e7b4 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -70,7 +70,7 @@ static void *sendRequest(void *param) { } int main(int argc, char *argv[]) { - SRpcInit rpcInit; + SRpcInit rpc_rpcInit; SRpcEpSet epSet; int msgSize = 128; int numOfReqs = 0; @@ -90,18 +90,18 @@ int main(int argc, char *argv[]) { strcpy(epSet.fqdn[1], "192.168.0.1"); // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; - rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; - rpcInit.user = "michael"; - rpcInit.secret = secret; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.connType = TAOS_CONN_CLIENT; + memset(&rpc_rpcInit, 0, sizeof(rpc_rpcInit)); + rpc_rpcInit.localPort = 0; + rpc_rpcInit.label = "APP"; + rpc_rpcInit.numOfThreads = 1; + rpc_rpcInit.cfp = processResponse; + rpc_rpcInit.sessions = 100; + rpc_rpcInit.idleTime = tsShellActivityTimer*1000; + rpc_rpcInit.user = "michael"; + rpc_rpcInit.secret = secret; + rpc_rpcInit.ckey = "key"; + rpc_rpcInit.spi = 1; + rpc_rpcInit.connType = TAOS_CONN_CLIENT; for (int i=1; ireplica; ++index) { - const SNodeInfo *pNodeInfo = pCfg->nodeInfo + index; - pNode->peerInfo[index] = syncAddPeer(pNode, pNodeInfo); - if (pNode->peerInfo[index] == NULL) { + for (int32_t syn_index = 0; syn_index < pCfg->replica; ++syn_index) { + const SNodeInfo *pNodeInfo = pCfg->nodeInfo + syn_index; + pNode->peerInfo[syn_index] = syncAddPeer(pNode, pNodeInfo); + if (pNode->peerInfo[syn_index] == NULL) { sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, pNodeInfo->nodeFqdn, pNodeInfo->nodePort); syncStop(pNode->rid); @@ -210,7 +210,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) { - pNode->selfIndex = index; + pNode->selfIndex = syn_index; } } @@ -256,8 +256,8 @@ int64_t syncStart(const SSyncInfo *pInfo) { } syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb - for (int32_t index = 0; index < pNode->replica; ++index) { - syncStartCheckPeerConn(pNode->peerInfo[index]); + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + syncStartCheckPeerConn(pNode->peerInfo[syn_index]); } return pNode->rid; @@ -277,8 +277,8 @@ void syncStop(int64_t rid) { if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer); if (pNode->pRoleTimer) taosTmrStop(pNode->pRoleTimer); - for (int32_t index = 0; index < pNode->replica; ++index) { - pPeer = pNode->peerInfo[index]; + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + pPeer = pNode->peerInfo[syn_index]; if (pPeer) syncRemovePeer(pPeer); } @@ -303,8 +303,8 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { pthread_mutex_lock(&pNode->mutex); syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb - for (int32_t index = 0; index < pNode->replica; ++index) { - syncStopCheckPeerConn(pNode->peerInfo[index]); + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + syncStopCheckPeerConn(pNode->peerInfo[syn_index]); } for (i = 0; i < pNode->replica; ++i) { @@ -364,8 +364,8 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { } syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb - for (int32_t index = 0; index < pNode->replica; ++index) { - syncStartCheckPeerConn(pNode->peerInfo[index]); + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + syncStartCheckPeerConn(pNode->peerInfo[syn_index]); } pthread_mutex_unlock(&pNode->mutex); @@ -629,16 +629,16 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { } void syncBroadcastStatus(SSyncNode *pNode) { - for (int32_t index = 0; index < pNode->replica; ++index) { - if (index == pNode->selfIndex) continue; - SSyncPeer *pPeer = pNode->peerInfo[index]; + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + if (syn_index == pNode->selfIndex) continue; + SSyncPeer *pPeer = pNode->peerInfo[syn_index]; syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_BROADCAST, syncGenTranId()); } } static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int32_t index = 0; index < pNode->replica; ++index) { - pNode->peerInfo[index]->numOfRetrieves = 0; + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + pNode->peerInfo[syn_index]->numOfRetrieves = 0; } if (pNode->notifyFlowCtrlFp) { @@ -649,7 +649,7 @@ static void syncResetFlowCtrl(SSyncNode *pNode) { static void syncChooseMaster(SSyncNode *pNode) { SSyncPeer *pPeer; int32_t onlineNum = 0; - int32_t index = -1; + int32_t syn_index = -1; int32_t replica = pNode->replica; for (int32_t i = 0; i < pNode->replica; ++i) { @@ -660,13 +660,13 @@ static void syncChooseMaster(SSyncNode *pNode) { if (onlineNum == pNode->replica) { // if all peers are online, peer with highest version shall be master - index = 0; + syn_index = 0; for (int32_t i = 1; i < pNode->replica; ++i) { - if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) { - index = i; + if (pNode->peerInfo[i]->version > pNode->peerInfo[syn_index]->version) { + syn_index = i; } } - sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index); + sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[syn_index]->id, syn_index); } else { sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica); } @@ -683,26 +683,26 @@ static void syncChooseMaster(SSyncNode *pNode) { } } - if (index < 0 && onlineNum > replica / 2.0) { + if (syn_index < 0 && onlineNum > replica / 2.0) { // over half of nodes are online for (int32_t i = 0; i < pNode->replica; ++i) { // slave with highest version shall be master pPeer = pNode->peerInfo[i]; if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) { - if (index < 0 || pPeer->version > pNode->peerInfo[index]->version) { - index = i; + if (syn_index < 0 || pPeer->version > pNode->peerInfo[syn_index]->version) { + syn_index = i; } } } - if (index >= 0) { + if (syn_index >= 0) { sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId, - pNode->peerInfo[index]->id, index, onlineNum, replica); + pNode->peerInfo[syn_index]->id, syn_index, onlineNum, replica); } } - if (index >= 0) { - if (index == pNode->selfIndex) { + if (syn_index >= 0) { + if (syn_index == pNode->selfIndex) { sInfo("vgId:%d, start to work as master", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; @@ -712,7 +712,7 @@ static void syncChooseMaster(SSyncNode *pNode) { syncResetFlowCtrl(pNode); (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } else { - pPeer = pNode->peerInfo[index]; + pPeer = pNode->peerInfo[syn_index]; sInfo("%s, it shall work as master", pPeer->id); } } else { @@ -725,8 +725,8 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { int32_t masterIndex = -1; int32_t replica = pNode->replica; - for (int32_t index = 0; index < pNode->replica; ++index) { - if (pNode->peerInfo[index]->role != TAOS_SYNC_ROLE_OFFLINE) { + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + if (pNode->peerInfo[syn_index]->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; } } @@ -751,19 +751,19 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } } else { - for (int32_t index = 0; index < pNode->replica; ++index) { - SSyncPeer *pTemp = pNode->peerInfo[index]; + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + SSyncPeer *pTemp = pNode->peerInfo[syn_index]; if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; if (masterIndex < 0) { - masterIndex = index; - sDebug("vgId:%d, peer:%s is master, index:%d", pNode->vgId, pTemp->id, index); + masterIndex = syn_index; + sDebug("vgId:%d, peer:%s is master, index:%d", pNode->vgId, pTemp->id, syn_index); } else { // multiple masters, it shall not happen if (masterIndex == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } else { - sError("vgId:%d, peer:%s is master too, masterIndex:%d index:%d", pNode->vgId, pTemp->id, masterIndex, index); + sError("vgId:%d, peer:%s is master too, masterIndex:%d index:%d", pNode->vgId, pTemp->id, masterIndex, syn_index); } } } @@ -783,9 +783,9 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) { (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); code = -1; - for (int32_t index = 0; index < pNode->replica; ++index) { - if (index == pNode->selfIndex) continue; - syncRestartPeer(pNode->peerInfo[index]); + for (int32_t syn_index = 0; syn_index < pNode->replica; ++syn_index) { + if (syn_index == pNode->selfIndex) continue; + syncRestartPeer(pNode->peerInfo[syn_index]); } } @@ -825,15 +825,15 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } else { // master not there, if all peer's state and version are consistent, choose the master int32_t consistent = 0; - int32_t index = 0; + int32_t syn_index = 0; if (peersStatus != NULL) { - for (index = 0; index < pNode->replica; ++index) { - SSyncPeer *pTemp = pNode->peerInfo[index]; - if (pTemp->role != peersStatus[index].role) break; - if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[index].version)) break; + for (syn_index = 0; syn_index < pNode->replica; ++syn_index) { + SSyncPeer *pTemp = pNode->peerInfo[syn_index]; + if (pTemp->role != peersStatus[syn_index].role) break; + if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[syn_index].version)) break; } - if (index >= pNode->replica) consistent = 1; + if (syn_index >= pNode->replica) consistent = 1; } else { if (pNode->replica == 2) consistent = 1; } @@ -1331,7 +1331,7 @@ static void syncProcessBrokenLink(int64_t rid, int32_t closedByApp) { static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t _version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; - int64_t time = taosGetTimestampMs(); + int64_t syn_time = taosGetTimestampMs(); if (pSyncFwds->fwds >= SYNC_MAX_FWDS) { // pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; @@ -1348,7 +1348,7 @@ static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t _version, void *mhandl memset(pFwdInfo, 0, sizeof(SFwdInfo)); pFwdInfo->version = _version; pFwdInfo->mhandle = mhandle; - pFwdInfo->time = time; + pFwdInfo->time = syn_time; pSyncFwds->fwds++; sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, _version, pSyncFwds->fwds); @@ -1400,10 +1400,10 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; - for (int32_t index = 0; index < pNode->replica; index++) { - if (index == pNode->selfIndex) continue; + for (int32_t syn_index = 0; syn_index < pNode->replica; syn_index++) { + if (syn_index == pNode->selfIndex) continue; - SSyncPeer *pPeer = pNode->peerInfo[index]; + SSyncPeer *pPeer = pNode->peerInfo[syn_index]; if (/*pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && */ nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; if (/*pPeer->sstatus > TAOS_SYNC_STATUS_INIT || */ nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; @@ -1425,16 +1425,16 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; if (pSyncFwds) { - int64_t time = taosGetTimestampMs(); + int64_t syn_time = taosGetTimestampMs(); if (pSyncFwds->fwds > 0) { pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS; - if (ABS(time - pFwdInfo->time) < 10000) break; + if (ABS(syn_time - pFwdInfo->time) < 10000) break; sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, - pFwdInfo->version, time, pFwdInfo->time); + pFwdInfo->version, syn_time, pFwdInfo->time); syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_SYN_CONFIRM_EXPIRED); } @@ -1502,7 +1502,10 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } } - int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); + SOCKET peerFd = pPeer->peerFd; + pthread_mutex_unlock(&pNode->mutex); + int32_t retLen = taosWriteMsg(peerFd, pSyncHead, fwdLen); + pthread_mutex_lock(&pNode->mutex); if (retLen == fwdLen) { sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index c86ab8549974712658ad3d381c4141427c000762..864f4c369644eccb21f96531dfda1daf51b401ad 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -228,12 +228,12 @@ static int64_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi return code; } -static int64_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { +static int64_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t syn_index) { SSyncNode *pNode = pPeer->pSyncNode; int32_t once = 0; // last WAL has once ever been processed int64_t offset = 0; uint64_t fversion = 0; - char fname[TSDB_FILENAME_LEN * 2] = {0}; // full path to wal file + char fname[TSDB_FILENAME_LEN * 3] = {0}; // full path to wal file // get full path to wal file snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); @@ -290,12 +290,12 @@ static int64_t syncRetrieveWal(SSyncPeer *pPeer) { char wname[TSDB_FILENAME_LEN * 2]; int32_t size; int64_t code = -1; - int64_t index = 0; + int64_t syn_index = 0; while (1) { // retrieve wal info wname[0] = 0; - code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index); + code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &syn_index); if (code < 0) { sError("%s, failed to get wal info since:%s, code:0x%" PRIx64, pPeer->id, strerror(errno), code); break; @@ -308,7 +308,7 @@ static int64_t syncRetrieveWal(SSyncPeer *pPeer) { } if (code == 0) { // last wal - code = syncProcessLastWal(pPeer, wname, index); + code = syncProcessLastWal(pPeer, wname, syn_index); sInfo("%s, last wal processed, code:%" PRId64, pPeer->id, code); break; } @@ -317,14 +317,14 @@ static int64_t syncRetrieveWal(SSyncPeer *pPeer) { snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); // send wal file, old wal file won't be modified, even remove is ok - struct stat fstat; - if (stat(fname, &fstat) < 0) { + struct stat syn_fstat; + if (stat(fname, &syn_fstat) < 0) { code = -1; sInfo("%s, failed to stat wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); break; } - size = fstat.st_size; + size = syn_fstat.st_size; sInfo("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); int32_t sfd = open(fname, O_RDONLY | O_BINARY); diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c index 23ea54ee0c19b6ad2f93d7577d8d711874b10968..b243c47fd32ef4056a42a701fc53a96632ac4da8 100644 --- a/src/sync/test/syncClient.c +++ b/src/sync/test/syncClient.c @@ -71,7 +71,7 @@ void *sendRequest(void *param) { } int main(int argc, char *argv[]) { - SRpcInit rpcInit; + SRpcInit syn_rpcInit; SRpcEpSet epSet; char secret[TSDB_KEY_LEN] = "mypassword"; int msgSize = 128; @@ -91,18 +91,18 @@ int main(int argc, char *argv[]) { strcpy(epSet.fqdn[1], "192.168.0.1"); // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; - rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; - rpcInit.user = "michael"; - rpcInit.secret = secret; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.connType = TAOS_CONN_CLIENT; + memset(&syn_rpcInit, 0, sizeof(syn_rpcInit)); + syn_rpcInit.localPort = 0; + syn_rpcInit.label = "APP"; + syn_rpcInit.numOfThreads = 1; + syn_rpcInit.cfp = processResponse; + syn_rpcInit.sessions = 100; + syn_rpcInit.idleTime = tsShellActivityTimer*1000; + syn_rpcInit.user = "michael"; + syn_rpcInit.secret = secret; + syn_rpcInit.ckey = "key"; + syn_rpcInit.spi = 1; + syn_rpcInit.connType = TAOS_CONN_CLIENT; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { @@ -110,11 +110,11 @@ int main(int argc, char *argv[]) { } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { tstrncpy(epSet.fqdn[0], argv[++i], TSDB_FQDN_LEN); } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { - rpcInit.numOfThreads = atoi(argv[++i]); + syn_rpcInit.numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { msgSize = atoi(argv[++i]); } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { - rpcInit.sessions = atoi(argv[++i]); + syn_rpcInit.sessions = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) { numOfReqs = atoi(argv[++i]); } else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) { @@ -122,26 +122,26 @@ int main(int argc, char *argv[]) { } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { tsCompressMsgSize = atoi(argv[++i]); } else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) { - rpcInit.user = argv[++i]; + syn_rpcInit.user = argv[++i]; } else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) { - rpcInit.secret = argv[++i]; + syn_rpcInit.secret = argv[++i]; } else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) { - rpcInit.spi = atoi(argv[++i]); + syn_rpcInit.spi = atoi(argv[++i]); } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { rpcDebugFlag = atoi(argv[++i]); } else { printf("\nusage: %s [options] \n", argv[0]); printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]); - printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); - printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); + printf(" [-t threads]: number of rpc threads, default is:%d\n", syn_rpcInit.numOfThreads); + printf(" [-s sessions]: number of rpc sessions, default is:%d\n", syn_rpcInit.sessions); printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); printf(" [-a threads]: number of app threads, default is:%d\n", appThreads); printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs); printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); - printf(" [-u user]: user name for the connection, default is:%s\n", rpcInit.user); - printf(" [-k secret]: password for the connection, default is:%s\n", rpcInit.secret); - printf(" [-spi SPI]: security parameter index, default is:%d\n", rpcInit.spi); + printf(" [-u user]: user name for the connection, default is:%s\n", syn_rpcInit.user); + printf(" [-k secret]: password for the connection, default is:%s\n", syn_rpcInit.secret); + printf(" [-spi SPI]: security parameter index, default is:%d\n", syn_rpcInit.spi); printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); printf(" [-h help]: print out this help\n\n"); exit(0); @@ -150,7 +150,7 @@ int main(int argc, char *argv[]) { taosInitLog("client.log", 100000, 10); - void *pRpc = rpcOpen(&rpcInit); + void *pRpc = rpcOpen(&syn_rpcInit); if (pRpc == NULL) { uError("failed to initialize RPC"); return -1; diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 4598e16a9d05be29d11612755a079ce0a228a2ff..a9d9af666775c87f0e2159bdcbc64b71a7e9a1ef 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -229,47 +229,47 @@ void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); } -uint32_t getFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { +uint32_t getFileInfo(int32_t vgId, char *name, uint32_t *syn_index, uint32_t eindex, int64_t *size, uint64_t *fversion) { uint32_t magic; - struct stat fstat; + struct stat syn_fstat; char aname[280]; - if (*index == 2) { + if (*syn_index == 2) { uInfo("wait for a while ....."); sleep(3); } if (name[0] == 0) { // find the file - snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *index); - sprintf(name, "data/data.%d", *index); + snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *syn_index); + sprintf(name, "data/data.%d", *syn_index); } else { snprintf(aname, sizeof(aname), "%s/%s", path, name); } uInfo("get file info:%s", aname); - if (stat(aname, &fstat) < 0) return 0; + if (stat(aname, &syn_fstat) < 0) return 0; - *size = fstat.st_size; - magic = fstat.st_size; + *size = syn_fstat.st_size; + magic = syn_fstat.st_size; return magic; } -int getWalInfo(int32_t vgId, char *name, int64_t *index) { - struct stat fstat; +int getWalInfo(int32_t vgId, char *name, int64_t *syn_index) { + struct stat syn_fstat; char aname[280]; name[0] = 0; - if (*index + 1 > walNum) return 0; + if (*syn_index + 1 > walNum) return 0; - snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index); - sprintf(name, "wal/wal.%d", *index); + snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *syn_index); + sprintf(name, "wal/wal.%d", *syn_index); uInfo("get wal info:%s", aname); - if (stat(aname, &fstat) < 0) return -1; + if (stat(aname, &syn_fstat) < 0) return -1; - if (*index >= walNum - 1) return 0; // no more + if (*syn_index >= walNum - 1) return 0; // no more return 1; } @@ -342,30 +342,30 @@ void doSync() { } int main(int argc, char *argv[]) { - SRpcInit rpcInit; + SRpcInit syn_rpcInit; char dataName[20] = "server.data"; pCfg = &syncInfo.syncCfg; initSync(); - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 7000; - rpcInit.label = "SER"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processRequestMsg; - rpcInit.sessions = 1000; - rpcInit.idleTime = tsShellActivityTimer * 1500; - rpcInit.afp = retrieveAuthInfo; + memset(&syn_rpcInit, 0, sizeof(syn_rpcInit)); + syn_rpcInit.localPort = 7000; + syn_rpcInit.label = "SER"; + syn_rpcInit.numOfThreads = 1; + syn_rpcInit.cfp = processRequestMsg; + syn_rpcInit.sessions = 1000; + syn_rpcInit.idleTime = tsShellActivityTimer * 1500; + syn_rpcInit.afp = retrieveAuthInfo; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - rpcInit.localPort = atoi(argv[++i]); + syn_rpcInit.localPort = atoi(argv[++i]); } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { - rpcInit.numOfThreads = atoi(argv[++i]); + syn_rpcInit.numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { msgSize = atoi(argv[++i]); } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { - rpcInit.sessions = atoi(argv[++i]); + syn_rpcInit.sessions = atoi(argv[++i]); } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { tsCompressMsgSize = atoi(argv[++i]); } else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) { @@ -380,9 +380,9 @@ int main(int argc, char *argv[]) { rpcDebugFlag = atoi(argv[++i]); } else { printf("\nusage: %s [options] \n", argv[0]); - printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); - printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); - printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); + printf(" [-p port]: server port number, default is:%d\n", syn_rpcInit.localPort); + printf(" [-t threads]: number of rpc threads, default is:%d\n", syn_rpcInit.numOfThreads); + printf(" [-s sessions]: number of sessions, default is:%d\n", syn_rpcInit.sessions); printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); @@ -401,14 +401,14 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; taosInitLog("server.log", 1000000, 10); - rpcInit.connType = TAOS_CONN_SERVER; - void *pRpc = rpcOpen(&rpcInit); + syn_rpcInit.connType = TAOS_CONN_SERVER; + void *pRpc = rpcOpen(&syn_rpcInit); if (pRpc == NULL) { uError("failed to start RPC server"); return -1; } - tsSyncPort = rpcInit.localPort + 10; + tsSyncPort = syn_rpcInit.localPort + 10; qhandle = taosOpenQueue(); doSync(); @@ -422,7 +422,7 @@ int main(int argc, char *argv[]) { return -1; } - printf("server is running, localPort:%d\n", rpcInit.localPort); + printf("server is running, localPort:%d\n", syn_rpcInit.localPort); SNodesRole nroles; while (1) { diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h index ec6b057aef142fb938993b3a27717c5e64937258..4b650d3993a54f6a98caf00a3605feb37e972ebd 100644 --- a/src/tsdb/inc/tsdbBuffer.h +++ b/src/tsdb/inc/tsdbBuffer.h @@ -29,6 +29,7 @@ typedef struct { int tBufBlocks; int nBufBlocks; int nRecycleBlocks; + int nElasticBlocks; int64_t index; SList* bufBlockList; } STsdbBufPool; @@ -41,6 +42,10 @@ int tsdbOpenBufPool(STsdbRepo* pRepo); void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode); +void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic); + +// health cite +STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize); +void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock); #endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbHealth.h b/src/tsdb/inc/tsdbHealth.h new file mode 100644 index 0000000000000000000000000000000000000000..324f4312e05fc0ca0200c319728bf692bf476bf6 --- /dev/null +++ b/src/tsdb/inc/tsdbHealth.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_HEALTH_H_ +#define _TD_TSDB_HEALTH_H_ + +bool tsdbUrgeQueryFree(STsdbRepo* pRepo); +int32_t tsdbInsertNewBlock(STsdbRepo* pRepo); + +bool tsdbIdleMemEnough(); +bool tsdbAllowNewBlock(STsdbRepo* pRepo); + +#endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h index a13b522eece382fc9c8f0d1de471e2f1c9421840..941b958c04beee4bda98275edd4a6b58e13eb32f 100644 --- a/src/tsdb/inc/tsdbMeta.h +++ b/src/tsdb/inc/tsdbMeta.h @@ -41,6 +41,7 @@ typedef struct STable { int16_t restoreColumnNum; bool hasRestoreLastColumn; int lastColSVersion; + int16_t cacheLastConfigVersion; T_REF_DECLARE() } STable; diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index 532907ae01be576e40feea2969761846f07170b3..a7f08e61ed1ac576c1ca70c0da20e14f4b3a306f 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -79,8 +79,8 @@ struct STsdbRepo { STsdbCfg save_config; // save apply config bool config_changed; // config changed flag pthread_mutex_t save_mutex; // protect save config - - uint8_t hasCachedLastColumn; + + int16_t cacheLastConfigVersion; STsdbAppH appH; STsdbStat stat; @@ -97,6 +97,7 @@ struct STsdbRepo { SMergeBuf mergeBuf; //used when update=2 int8_t compactState; // compact state: inCompact/noCompact/waitingCompact? + pthread_t* pthread; }; #define REPO_ID(r) (r)->config.tsdbId @@ -110,7 +111,8 @@ int tsdbUnlockRepo(STsdbRepo* pRepo); STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo); int tsdbCheckCommit(STsdbRepo* pRepo); int tsdbRestoreInfo(STsdbRepo* pRepo); -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg); +UNUSED_FUNC int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg); +int32_t tsdbLoadLastCache(STsdbRepo *pRepo, STable* pTable); void tsdbGetRootDir(int repoid, char dirName[]); void tsdbGetDataDir(int repoid, char dirName[]); diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index e675bf6f9de04021112d43a1db70cf56cf430f08..1faec90bf6fcb0fa48ff0b539e2c6da7cbe6df91 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -14,11 +14,10 @@ */ #include "tsdbint.h" +#include "tsdbHealth.h" #define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0) -static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize); -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock); // ---------------- INTERNAL FUNCTIONS ---------------- STsdbBufPool *tsdbNewBufPool() { @@ -69,6 +68,7 @@ int tsdbOpenBufPool(STsdbRepo *pRepo) { pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB pPool->tBufBlocks = pCfg->totalBlocks; pPool->nBufBlocks = 0; + pPool->nElasticBlocks = 0; pPool->index = 0; pPool->nRecycleBlocks = 0; @@ -120,6 +120,18 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { STsdbBufPool *pBufPool = pRepo->pPool; while (POOL_IS_EMPTY(pBufPool)) { + if(tsDeadLockKillQuery) { + // supply new Block + if(tsdbInsertNewBlock(pRepo) > 0) { + tsdbWarn("vgId:%d add new elastic block . elasticBlocks=%d cur free Blocks=%d", REPO_ID(pRepo), pBufPool->nElasticBlocks, pBufPool->bufBlockList->numOfEles); + break; + } else { + // no newBlock, kill query free + if(!tsdbUrgeQueryFree(pRepo)) + tsdbWarn("vgId:%d Urge query free thread start failed.", REPO_ID(pRepo)); + } + } + pRepo->repoLocked = false; pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex)); pRepo->repoLocked = true; @@ -139,11 +151,11 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { } // ---------------- LOCAL FUNCTIONS ---------------- -static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { +STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize); if (pBufBlock == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + return NULL; } pBufBlock->blockId = 0; @@ -151,13 +163,9 @@ static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { pBufBlock->remain = bufBlockSize; return pBufBlock; - -_err: - tsdbFreeBufBlock(pBufBlock); - return NULL; } -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } +void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { if (oldTotalBlocks == pRepo->config.totalBlocks) { @@ -193,10 +201,15 @@ err: return err; } -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode) { +void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic) { STsdbBufBlock *pBufBlock = NULL; tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock)); tsdbFreeBufBlock(pBufBlock); free(pNode); - pPool->nBufBlocks--; + if(bELastic) { + pPool->nElasticBlocks--; + tsdbWarn("pPool=%p elastic block reduce one . nElasticBlocks=%d cur free Blocks=%d", pPool, pPool->nElasticBlocks, pPool->bufBlockList->numOfEles); + } + else + pPool->nBufBlocks--; } diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index adb7fae0ab7e81eb2718e13eee8485192644de74..6790e93782c312dae125cafd7821347b032983f1 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -223,7 +223,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { SBlockIdx *pBlkIdx; size_t nidx = taosArrayGetSize(pIdxA); int tlen = 0, size; - int64_t offset; + int64_t offset = 0; if (nidx <= 0) { // All data are deleted diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index 59fb4f334d3006eb7e8807ce193d61905f2322d2..dccb85af55b8f76739f16621a0959c29fb373b4a 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -146,7 +146,9 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { if (oldCfg.cacheLastRow != pRepo->config.cacheLastRow) { if (tsdbLockRepo(pRepo) < 0) return; - tsdbCacheLastData(pRepo, &oldCfg); + // tsdbCacheLastData(pRepo, &oldCfg); + // lazy load last cache when query or update + ++pRepo->cacheLastConfigVersion; tsdbUnlockRepo(pRepo); } diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 9b890ec113600420c179f4d13159706d79dd322a..a898b1f7ee8ea8806880b69d34224153e6322210 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -64,7 +64,9 @@ int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); } void *tsdbCompactImpl(STsdbRepo *pRepo) { // Check if there are files in TSDB FS to compact if (REPO_FS(pRepo)->cstatus->pmf == NULL) { - tsdbInfo("vgId:%d no file to compact in FS", REPO_ID(pRepo)); + pRepo->compactState = TSDB_NO_COMPACT; + tsem_post(&(pRepo->readyToCommit)); + tsdbInfo("vgId:%d compact over, no file to compact in FS", REPO_ID(pRepo)); return NULL; } diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 70a52621383ec5e1099a815dceefb0b4493c1f9e..22de81c4fe99f97bb3001ac1884d5b2e03efab88 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -1164,10 +1164,10 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { // Sort the array according to file name taosArraySort(fArray, tsdbComparTFILE); - size_t index = 0; + size_t tsd_index = 0; // Loop to recover each file set for (;;) { - if (index >= taosArrayGetSize(fArray)) { + if (tsd_index >= taosArrayGetSize(fArray)) { break; } @@ -1179,13 +1179,13 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); - if (index >= taosArrayGetSize(fArray)) { + if (tsd_index >= taosArrayGetSize(fArray)) { tsdbError("vgId:%d incomplete DFileSet, fid:%d", REPO_ID(pRepo), fset.fid); taosArrayDestroy(fArray); return -1; } - pf = taosArrayGet(fArray, index); + pf = taosArrayGet(fArray, tsd_index); int tvid, tfid; TSDB_FILE_T ttype; @@ -1198,7 +1198,7 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { ASSERT(tvid == REPO_ID(pRepo)); if (tfid < pRepo->rtn.minFid) { // skip file expired - ++index; + ++tsd_index; continue; } @@ -1252,7 +1252,7 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { } tsdbCloseDFile(pDFile); - index++; + tsd_index++; } tsdbInfo("vgId:%d FSET %d is restored", REPO_ID(pRepo), fset.fid); diff --git a/src/tsdb/src/tsdbHealth.c b/src/tsdb/src/tsdbHealth.c new file mode 100644 index 0000000000000000000000000000000000000000..8198c480334912b1ce373ceca7b82409f5a644f2 --- /dev/null +++ b/src/tsdb/src/tsdbHealth.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosmsg.h" +#include "tarray.h" +#include "query.h" +#include "tglobal.h" +#include "tlist.h" +#include "tsdbint.h" +#include "tsdbBuffer.h" +#include "tsdbLog.h" +#include "tsdbHealth.h" +#include "ttimer.h" +#include "tthread.h" + + +// return malloc new block count +int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { + STsdbBufPool *pPool = pRepo->pPool; + int32_t cnt = 0; + + if(tsdbAllowNewBlock(pRepo)) { + STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); + if (pBufBlock) { + if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { + // append error + tsdbFreeBufBlock(pBufBlock); + } else { + pPool->nElasticBlocks ++; + cnt ++ ; + } + } + } + return cnt; +} + +// switch anther thread to run +void* cbKillQueryFree(void* param) { + STsdbRepo* pRepo = (STsdbRepo*)param; + // vnode + if(pRepo->appH.notifyStatus) { + pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); + } + + // free + if(pRepo->pthread){ + void* p = pRepo->pthread; + pRepo->pthread = NULL; + free(p); + } + + return NULL; +} + +// return true do free , false do nothing +bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { + // check previous running + if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { + tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); + return false; + } + // create new + pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); + if(pRepo->pthread == NULL) { + tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); + return false; + } + return true; +} + +bool tsdbAllowNewBlock(STsdbRepo* pRepo) { + int32_t nMaxElastic = pRepo->config.totalBlocks/3; + STsdbBufPool* pPool = pRepo->pPool; + if(pPool->nElasticBlocks >= nMaxElastic) { + tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); + return false; + } + return true; +} + +bool tsdbNoProblem(STsdbRepo* pRepo) { + if(listNEles(pRepo->pPool->bufBlockList) == 0) + return false; + return true; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 9cb04738534da529e7f31b4ec39f473b6e722061..9cd3993ea85708a325432147b6f7f055edf61c03 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -16,6 +16,8 @@ // no test file errors here #include "taosdef.h" #include "tsdbint.h" +#include "ttimer.h" +#include "tthread.h" #define IS_VALID_PRECISION(precision) \ (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) @@ -126,6 +128,10 @@ int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { terrno = TSDB_CODE_SUCCESS; tsdbStopStream(pRepo); + if(pRepo->pthread){ + taosDestoryThread(pRepo->pthread); + pRepo->pthread = NULL; + } if (toCommit) { tsdbSyncCommit(repo); @@ -330,7 +336,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { #endif } -uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { +uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *tsd_index, uint32_t eindex, int64_t *size) { // TODO return 0; #if 0 @@ -342,16 +348,16 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t struct stat fState; - tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); - ASSERT(*index <= eindex); + tsdbDebug("vgId:%d name:%s tsd_index:%d eindex:%d", pRepo->config.tsdbId, name, *tsd_index, eindex); + ASSERT(*tsd_index <= eindex); if (name[0] == 0) { // get the file from index or after, but not larger than eindex - int fid = (*index) / TSDB_FILE_TYPE_MAX; + int fid = (*tsd_index) / TSDB_FILE_TYPE_MAX; if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { - if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { + if (*tsd_index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { fname = tsdbGetMetaFileName(pRepo->rootDir); - *index = TSDB_META_FILE_INDEX; + *tsd_index = TSDB_META_FILE_INDEX; magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); } else { @@ -361,7 +367,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t SFileGroup *pFGroup = taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { - SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; + SFile *pFile = &pFGroup->files[(*tsd_index) % TSDB_FILE_TYPE_MAX]; fname = strdup(TSDB_FILE_NAME(pFile)); magic = pFile->info.magic; char *tfname = strdup(fname); @@ -371,7 +377,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { SFile *pFile = &pFGroup->files[0]; fname = strdup(TSDB_FILE_NAME(pFile)); - *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + *tsd_index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; magic = pFile->info.magic; char *tfname = strdup(fname); sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); @@ -388,7 +394,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t tfree(fname); return 0; } - if (*index == TSDB_META_FILE_INDEX) { // get meta file + if (*tsd_index == TSDB_META_FILE_INDEX) { // get meta file tsdbGetStoreInfo(fname, &magic, size); } else { char tfname[TSDB_FILENAME_LEN] = "\0"; @@ -499,6 +505,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } // Check keep +#if 0 // already checked and set in mnodeSetDefaultDbCfg if (pCfg->keep == -1) { pCfg->keep = TSDB_DEFAULT_KEEP; } else { @@ -519,6 +526,25 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (pCfg->keep2 == 0) { pCfg->keep2 = pCfg->keep; } +#endif + + int32_t keepMin = pCfg->keep1; + int32_t keepMid = pCfg->keep2; + int32_t keepMax = pCfg->keep; + + if (keepMin > keepMid) { + SWAP(keepMin, keepMid, int32_t); + } + if (keepMin > keepMax) { + SWAP(keepMin, keepMax, int32_t); + } + if (keepMid > keepMax) { + SWAP(keepMid, keepMax, int32_t); + } + + pCfg->keep = keepMax; + pCfg->keep1 = keepMin; + pCfg->keep2 = keepMid; // update check if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE) @@ -547,6 +573,7 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { pRepo->appH = *pAppH; } pRepo->repoLocked = false; + pRepo->pthread = NULL; int code = pthread_mutex_init(&(pRepo->mutex), NULL); if (code != 0) { @@ -562,7 +589,7 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { return NULL; } pRepo->config_changed = false; - atomic_store_8(&pRepo->hasCachedLastColumn, 0); + pRepo->cacheLastConfigVersion = 0; code = tsem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { @@ -646,9 +673,9 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea int numColumns; int32_t blockIdx; SDataStatis* pBlockStatis = NULL; - SMemRow row = NULL; + // SMemRow row = NULL; // restore last column data with last schema - + int err = 0; numColumns = schemaNCols(pSchema); @@ -662,17 +689,17 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea } } - row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (row == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = -1; - goto out; - } + // row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); + // if (row == NULL) { + // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + // err = -1; + // goto out; + // } - memRowSetType(row, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(row), pSchema); + // memRowSetType(row, SMEM_ROW_DATA); + // tdInitDataRow(memRowDataBody(row), pSchema); - // first load block index info + // first load block tsd_index info if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { err = -1; goto out; @@ -711,7 +738,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); loadStatisData = true; } - + TSDB_WLOCK_TABLE(pTable); // lock when update pTable->lastCols[] for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { STColumn *pCol = schemaColAt(pSchema, i); // ignore loaded columns @@ -728,10 +755,11 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; const void* pColData = tdGetColDataOfRow(pDataCol, rowId); - tdAppendColVal(memRowDataBody(row), pColData, pCol->type, pCol->offset); - //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); - if (isNull(value, pCol->type)) { + // tdAppendColVal(memRowDataBody(row), pColData, pCol->type, pCol->offset); + // SDataCol *pDataCol = readh.pDCols[0]->cols + j; + // void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + + // pCol->offset); + if (isNull(pColData, pCol->type)) { continue; } @@ -746,24 +774,26 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea pLastCol->pData = malloc(bytes); pLastCol->bytes = bytes; pLastCol->colId = pCol->colId; - memcpy(pLastCol->pData, value, bytes); + memcpy(pLastCol->pData, pColData, bytes); // save row ts(in column 0) pDataCol = pReadh->pDCols[0]->cols + 0; - pCol = schemaColAt(pSchema, 0); - tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); - pLastCol->ts = memRowKey(row); - + // pCol = schemaColAt(pSchema, 0); + // tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); + // pLastCol->ts = memRowKey(row); + pLastCol->ts = tdGetKey(*(TKEY *)(tdGetColDataOfRow(pDataCol, rowId))); + pTable->restoreColumnNum += 1; tsdbDebug("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d, %" PRId64, REPO_ID(pRepo), pTable->name->data, pLastCol->colId, pLastCol->ts); break; } } + TSDB_WUNLOCK_TABLE(pTable); } out: - taosTZfree(row); + // taosTZfree(row); tfree(pBlockStatis); if (err == 0 && numColumns <= pTable->restoreColumnNum) { @@ -774,7 +804,6 @@ out: } static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx) { - ASSERT(pTable->lastRow == NULL); if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { return -1; } @@ -788,21 +817,32 @@ static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, // Get the data in row STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (pTable->lastRow == NULL) { + SMemRow lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); + if (lastRow == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } - memRowSetType(pTable->lastRow, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); + memRowSetType(lastRow, SMEM_ROW_DATA); + tdInitDataRow(memRowDataBody(lastRow), pSchema); for (int icol = 0; icol < schemaNCols(pSchema); icol++) { STColumn *pCol = schemaColAt(pSchema, icol); SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; - tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, + tdAppendColVal(memRowDataBody(lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->offset); } - pTable->lastKey = memRowKey(pTable->lastRow); + TSKEY lastKey = memRowKey(lastRow); + + // during the load data in file, new data would be inserted and last row has been updated + TSDB_WLOCK_TABLE(pTable); + if (pTable->lastRow == NULL) { + pTable->lastKey = lastKey; + pTable->lastRow = lastRow; + TSDB_WUNLOCK_TABLE(pTable); + } else { + TSDB_WUNLOCK_TABLE(pTable); + taosTZfree(lastRow); + } return 0; } @@ -874,14 +914,105 @@ int tsdbRestoreInfo(STsdbRepo *pRepo) { tsdbDestroyReadH(&readh); - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); + // if (CACHE_LAST_NULL_COLUMN(pCfg)) { + // atomic_store_8(&pRepo->hasCachedLastColumn, 1); + // } + + return 0; +} + +int32_t tsdbLoadLastCache(STsdbRepo *pRepo, STable *pTable) { + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + int cacheLastRowTableNum = 0; + int cacheLastColTableNum = 0; + + bool cacheLastRow = CACHE_LAST_ROW(&(pRepo->config)); + bool cacheLastCol = CACHE_LAST_NULL_COLUMN(&(pRepo->config)); + + tsdbDebug("tsdbLoadLastCache for %s, cacheLastRow:%d, cacheLastCol:%d", pTable->name->data, cacheLastRow, cacheLastCol); + + pTable->cacheLastConfigVersion = pRepo->cacheLastConfigVersion; + + if (!cacheLastRow && pTable->lastRow != NULL) { + taosTZfree(pTable->lastRow); + pTable->lastRow = NULL; + } + if (!cacheLastCol && pTable->lastCols != NULL) { + tsdbFreeLastColumns(pTable); + } + + if (!cacheLastRow && !cacheLastCol) { + return 0; + } + + cacheLastRowTableNum = (cacheLastRow && pTable->lastRow == NULL) ? 1 : 0; + cacheLastColTableNum = (cacheLastCol && pTable->lastCols == NULL) ? 1 : 0; + + if (cacheLastRowTableNum == 0 && cacheLastColTableNum == 0) { + return 0; } + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } + + tsdbRLockFS(REPO_FS(pRepo)); + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + while ((cacheLastRowTableNum > 0 || cacheLastColTableNum > 0) && (pSet = tsdbFSIterNext(&fsiter)) != NULL) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbUnLockFS(REPO_FS(pRepo)); + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbUnLockFS(REPO_FS(pRepo)); + tsdbDestroyReadH(&readh); + return -1; + } + + // tsdbDebug("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); + + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbUnLockFS(REPO_FS(pRepo)); + tsdbDestroyReadH(&readh); + return -1; + } + + SBlockIdx *pIdx = readh.pBlkIdx; + + if (pIdx && (cacheLastRowTableNum > 0) && (pTable->lastRow == NULL)) { + if (tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { + tsdbUnLockFS(REPO_FS(pRepo)); + tsdbDestroyReadH(&readh); + return -1; + } + cacheLastRowTableNum -= 1; + } + + // restore NULL columns + if (pIdx && (cacheLastColTableNum > 0) && !pTable->hasRestoreLastColumn) { + if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { + tsdbUnLockFS(REPO_FS(pRepo)); + tsdbDestroyReadH(&readh); + return -1; + } + if (pTable->hasRestoreLastColumn) { + cacheLastColTableNum -= 1; + } + } + } + + tsdbUnLockFS(REPO_FS(pRepo)); + tsdbDestroyReadH(&readh); + return 0; } -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { +UNUSED_FUNC int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { bool cacheLastRow = false, cacheLastCol = false; SFSIter fsiter; SReadH readh; @@ -915,9 +1046,9 @@ int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { // if close last option,need to free data if (need_free_last_row || need_free_last_col) { - if (need_free_last_col) { - atomic_store_8(&pRepo->hasCachedLastColumn, 0); - } + // if (need_free_last_col) { + // atomic_store_8(&pRepo->hasCachedLastColumn, 0); + // } tsdbInfo("free cache last data since cacheLast option changed"); for (int i = 1; i <= maxTableIdx; i++) { STable *pTable = pMeta->tables[i]; @@ -995,9 +1126,9 @@ int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { tsdbDestroyReadH(&readh); - if (cacheLastCol) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); - } + // if (cacheLastCol) { + // atomic_store_8(&pRepo->hasCachedLastColumn, 1); + // } return 0; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 54211d79f3836db9a0daca365c85616648ec53ca..be074bff83ec1c3fb5708fad6aac0afa33684a0e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -99,17 +99,22 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { STsdbBufPool *pBufPool = pRepo->pPool; SListNode *pNode = NULL; - bool recycleBlocks = pBufPool->nRecycleBlocks > 0; + bool addNew = false; if (tsdbLockRepo(pRepo) < 0) return -1; while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { if (pBufPool->nRecycleBlocks > 0) { - tsdbRecycleBufferBlock(pBufPool, pNode); + tsdbRecycleBufferBlock(pBufPool, pNode, false); pBufPool->nRecycleBlocks -= 1; } else { - tdListAppendNode(pBufPool->bufBlockList, pNode); + if(pBufPool->nElasticBlocks > 0 && listNEles(pBufPool->bufBlockList) > 2) { + tsdbRecycleBufferBlock(pBufPool, pNode, true); + } else { + tdListAppendNode(pBufPool->bufBlockList, pNode); + addNew = true; + } } } - if (!recycleBlocks) { + if (addNew) { int code = pthread_cond_signal(&pBufPool->poolNotEmpty); if (code != 0) { if (tsdbUnlockRepo(pRepo) < 0) return -1; @@ -555,7 +560,7 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); } +static char *tsdbGetTsTupleKey(const void *data) { return memRowKeys((SMemRow)data); } static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { ASSERT(pMemTable->maxTables < maxTables); @@ -996,7 +1001,8 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro if ((value == NULL) || isNull(value, pTCol->type)) { continue; } - + // lock + TSDB_WLOCK_TABLE(pTable); SDataCol *pDataCol = &(pLatestCols[idx]); if (pDataCol->pData == NULL) { pDataCol->pData = malloc(pTCol->bytes); @@ -1012,6 +1018,8 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro memcpy(pDataCol->pData, value, bytes); //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData); pDataCol->ts = memRowKey(row); + // unlock + TSDB_WUNLOCK_TABLE(pTable); } } @@ -1058,5 +1066,8 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow r updateTableLatestColumn(pRepo, pTable, row); } } + + pTable->cacheLastConfigVersion = pRepo->cacheLastConfigVersion; + return 0; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index e604b07cc1dd0d37574235263a018baecbe670c1..c1ccff8bb01116441fbdd4b156a8077a862fa89a 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -26,13 +26,13 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rm static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper); static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); -static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetName(STableCfg *config, char *name, bool dup); -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); +static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool tsdb_dup); +static int tsdbTableSetName(STableCfg *config, char *name, bool tsdb_dup); +static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool tsdb_dup); +static int tsdbTableSetSName(STableCfg *config, char *sname, bool tsdb_dup); static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); -static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); +static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool tsdb_dup); +static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool tsdb_dup); static int tsdbEncodeTableName(void **buf, tstr *name); static void * tsdbDecodeTableName(void *buf, tstr **name); static int tsdbEncodeTable(void **buf, STable *pTable); @@ -346,7 +346,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { tsdbError( "vgId:%d failed to update tag value of table %s since version out of date, client tag version %d server tag " "version %d", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), pMsg->tversion, schemaVersion(pTable->tagSchema)); + REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), pMsg->tversion, schemaVersion(pTable->pSuper->tagSchema)); terrno = TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; return -1; } @@ -627,27 +627,30 @@ int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) { } int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema) { - ASSERT(pTable->lastCols == NULL); + TSDB_WLOCK_TABLE(pTable); + if (pTable->lastCols == NULL) { + int16_t numOfColumn = pSchema->numOfCols; - int16_t numOfColumn = pSchema->numOfCols; + pTable->lastCols = (SDataCol *)malloc(numOfColumn * sizeof(SDataCol)); + if (pTable->lastCols == NULL) { + TSDB_WUNLOCK_TABLE(pTable); + return -1; + } - pTable->lastCols = (SDataCol*)malloc(numOfColumn * sizeof(SDataCol)); - if (pTable->lastCols == NULL) { - return -1; - } + for (int16_t i = 0; i < numOfColumn; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + SDataCol *pDataCol = &(pTable->lastCols[i]); + pDataCol->bytes = 0; + pDataCol->pData = NULL; + pDataCol->colId = pCol->colId; + } - for (int16_t i = 0; i < numOfColumn; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - SDataCol* pDataCol = &(pTable->lastCols[i]); - pDataCol->bytes = 0; - pDataCol->pData = NULL; - pDataCol->colId = pCol->colId; + pTable->lastColSVersion = schemaVersion(pSchema); + pTable->maxColNum = numOfColumn; + pTable->restoreColumnNum = 0; + pTable->hasRestoreLastColumn = false; } - - pTable->lastColSVersion = schemaVersion(pSchema); - pTable->maxColNum = numOfColumn; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; + TSDB_WUNLOCK_TABLE(pTable); return 0; } @@ -797,6 +800,7 @@ static STable *tsdbNewTable() { pTable->lastCols = NULL; pTable->restoreColumnNum = 0; + pTable->cacheLastConfigVersion = 0; pTable->maxColNum = 0; pTable->hasRestoreLastColumn = false; pTable->lastColSVersion = -1; @@ -1085,8 +1089,8 @@ static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, in return 0; } -static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (dup) { +static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool tsdb_dup) { + if (tsdb_dup) { config->schema = tdDupSchema(pSchema); if (config->schema == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1098,8 +1102,8 @@ static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) { return 0; } -static int tsdbTableSetName(STableCfg *config, char *name, bool dup) { - if (dup) { +static int tsdbTableSetName(STableCfg *config, char *name, bool tsdb_dup) { + if (tsdb_dup) { config->name = strdup(name); if (config->name == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1112,13 +1116,13 @@ static int tsdbTableSetName(STableCfg *config, char *name, bool dup) { return 0; } -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) { +static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool tsdb_dup) { if (config->type != TSDB_CHILD_TABLE) { terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; return -1; } - if (dup) { + if (tsdb_dup) { config->tagSchema = tdDupSchema(pSchema); if (config->tagSchema == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1130,13 +1134,13 @@ static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) return 0; } -static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) { +static int tsdbTableSetSName(STableCfg *config, char *sname, bool tsdb_dup) { if (config->type != TSDB_CHILD_TABLE) { terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; return -1; } - if (dup) { + if (tsdb_dup) { config->sname = strdup(sname); if (config->sname == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1158,13 +1162,13 @@ static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { return 0; } -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { +static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool tsdb_dup) { if (config->type != TSDB_CHILD_TABLE) { terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; return -1; } - if (dup) { + if (tsdb_dup) { config->tagValues = tdKVRowDup(row); if (config->tagValues == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1177,13 +1181,13 @@ static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { return 0; } -static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { +static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool tsdb_dup) { if (config->type != TSDB_STREAM_TABLE) { terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; return -1; } - if (dup) { + if (tsdb_dup) { config->sql = strdup(sql); if (config->sql == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index e6cf07ce7f7862848222d1dd4f1dd3114e42e130..f15a59cb626dcbe2d3be9348e0d95ca1414eee74 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -162,6 +162,7 @@ typedef struct STableGroupSupporter { static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); +static int32_t lazyLoadCacheLast(STsdbQueryHandle* pQueryHandle); static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); @@ -601,6 +602,28 @@ void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCon pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); } +static int32_t lazyLoadCacheLast(STsdbQueryHandle* pQueryHandle) { + STsdbRepo* pRepo = pQueryHandle->pTsdb; + + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t code = 0; + for (size_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STable* pTable = pCheckInfo->pTableObj; + if (pTable->cacheLastConfigVersion == pRepo->cacheLastConfigVersion) { + continue; + } + code = tsdbLoadLastCache(pRepo, pTable); + if (code != 0) { + tsdbError("%p uid:%" PRId64 ", tid:%d, failed to load last cache since %s", pQueryHandle, pTable->tableId.uid, + pTable->tableId.tid, tstrerror(terrno)); + break; + } + } + + return code; +} + TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); @@ -614,6 +637,8 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable return NULL; } + lazyLoadCacheLast(pQueryHandle); + int32_t code = checkForCachedLastRow(pQueryHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; @@ -628,13 +653,14 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable return pQueryHandle; } - TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); if (pQueryHandle == NULL) { return NULL; } + lazyLoadCacheLast(pQueryHandle); + int32_t code = checkForCachedLast(pQueryHandle); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; @@ -1054,7 +1080,7 @@ static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, uint if(ASCENDING_TRAVERSE(q->order)) { for(int32_t i = 0; i < numBlocks; i++) { SBlock* pBlock = &blocks[i]; - if(i == 0 && pBlock->keyFirst != s) { + if(i == 0 && pBlock->keyFirst != s) { //first block only keyFirst == s can skip q->frows += pBlock->numOfRows; continue; @@ -1071,7 +1097,7 @@ static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, uint } else { // des for(int32_t i = numBlocks - 1; i >= 0; i--) { SBlock* pBlock = &blocks[i]; - if(i == numBlocks - 1 && pBlock->keyLast != e) { + if(i == numBlocks - 1 && pBlock->keyLast != e) { //first block only keyFirst == s can skipƒ q->frows += pBlock->numOfRows; continue; @@ -1084,16 +1110,16 @@ static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, uint break; } } - } + } } return num; } -static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { +static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t tsd_index, int32_t* numOfBlocks) { int32_t code = 0; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, tsd_index); pCheckInfo->numOfBlocks = 0; if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { @@ -1433,66 +1459,63 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, return code; } -static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { - int firstPos, lastPos, midPos = -1; - int numOfRows; - TSKEY* keyList; - - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - if (num <= 0) return -1; - - keyList = (TSKEY*)pValue; - firstPos = 0; - lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { +// search last keyList[ret] < key order asc and keyList[ret] > key order desc +static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) { + // start end posistion + int s, e; + s = pos; + + // check + assert(pos >=0 && pos < num); + assert(num > 0); + + if (order == TSDB_ORDER_ASC) { // find the first position which is smaller than the key + e = num - 1; + if (key < keyList[pos]) + return -1; while (1) { - if (key >= keyList[lastPos]) return lastPos; - if (key == keyList[firstPos]) return firstPos; - if (key < keyList[firstPos]) return firstPos - 1; - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - - } else { + // check can return + if (key >= keyList[e]) + return e; + if (key <= keyList[s]) + return s; + if (e - s <= 1) + return s; + + // change start or end position + int mid = s + (e - s + 1)/2; + if (keyList[mid] > key) + e = mid; + else if(keyList[mid] < key) + s = mid; + else + return mid; + } + } else { // DESC // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; + e = 0; + if (key > keyList[pos]) + return -1; + while (1) { + // check can return + if (key <= keyList[e]) + return e; + if (key >= keyList[s]) + return s; + if (s - e <= 1) + return s; + + // change start or end position + int mid = s - (s - e + 1)/2; + if (keyList[mid] < key) + e = mid; + else if(keyList[mid] > key) + s = mid; else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } + return mid; + } } - } - - return midPos; } static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { @@ -1600,7 +1623,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int16_t offset; bool isRow1DataRow = isDataRow(row1); - bool isRow2DataRow; + bool isRow2DataRow = false; bool isChosenRowDataRow; int32_t chosen_itr; void *value; @@ -1900,7 +1923,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; - int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; SQueryFilePos* cur = &pQueryHandle->cur; SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; @@ -1913,7 +1935,9 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); } else { assert(pCols->numOfRows > 0); - endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); + int pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0 : pBlockInfo->rows - 1; + endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pos, pQueryHandle->window.ekey, pQueryHandle->order); + assert(endPos != -1); cur->mixBlock = true; } @@ -1933,17 +1957,16 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && cur->pos >= 0 && cur->pos < pBlock->numOfRows); - TSKEY* tsArray = pCols->cols[0].pData; - assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows-1] == pBlock->keyLast); - - // for search the endPos, so the order needs to reverse - int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + // key read from file + TSKEY* keyFile = pCols->cols[0].pData; + assert(pCols->numOfRows == pBlock->numOfRows && keyFile[0] == pBlock->keyFirst && keyFile[pBlock->numOfRows-1] == pBlock->keyLast); int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); STable* pTable = pCheckInfo->pTableObj; int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); + tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," "end:%d, 0x%"PRIx64, @@ -1958,6 +1981,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* STSchema* pSchema1 = NULL; STSchema* pSchema2 = NULL; + // position in file ->fpos int32_t pos = cur->pos; cur->win = TSWINDOW_INITIALIZER; @@ -1974,19 +1998,23 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* break; } - TSKEY key = memRowKey(row1); - if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + TSKEY keyMem = memRowKey(row1); + if ((keyMem > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (keyMem < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { break; } - if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - break; + // break if pos not in this block endPos range. note old code when pos is -1 can crash. + if(ASCENDING_TRAVERSE(pQueryHandle->order)) { //ASC + if(pos > endPos || keyFile[pos] > pQueryHandle->window.ekey) + break; + } else { //DESC + if(pos < endPos || keyFile[pos] < pQueryHandle->window.ekey) + break; } - if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((keyMem < keyFile[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (keyMem > keyFile[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (rv1 != memRowVersion(row1)) { pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1), (int8_t)memRowType(row1)); rv1 = memRowVersion(row1); @@ -1998,16 +2026,18 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); numOfRows += 1; + // record start key with memory key if not if (cur->win.skey == TSKEY_INITIAL_VAL) { - cur->win.skey = key; + cur->win.skey = keyMem; } - cur->win.ekey = key; - cur->lastKey = key + step; + cur->win.ekey = keyMem; + cur->lastKey = keyMem + step; cur->mixBlock = true; moveToNextRowInMem(pCheckInfo); - } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it + // same select mem key if update is true + } else if (keyMem == keyFile[pos]) { if (pCfg->update) { if(pCfg->update == TD_ROW_PARTIAL_UPDATE) { doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, pos, pos); @@ -2025,31 +2055,36 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { - cur->win.skey = key; + cur->win.skey = keyMem; } - cur->win.ekey = key; - cur->lastKey = key + step; + cur->win.ekey = keyMem; + cur->lastKey = keyMem + step; cur->mixBlock = true; - + + //mem move next moveToNextRowInMem(pCheckInfo); + //file move next, discard file row pos += step; } else { + // not update, only mem move to next, discard mem row moveToNextRowInMem(pCheckInfo); } - } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + // put file row + } else if ((keyMem > keyFile[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (keyMem < keyFile[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { - cur->win.skey = tsArray[pos]; + cur->win.skey = keyFile[pos]; } - int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); + int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pos, keyMem, pQueryHandle->order); assert(end != -1); - if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it + if (keyFile[end] == keyMem) { // the value of key in cache equals to the end timestamp value, ignore it if (pCfg->update == TD_ROW_DISCARD_UPDATE) { moveToNextRowInMem(pCheckInfo); } else { + // can update, don't copy then deal on next loop with keyMem == keyFile[pos] end -= step; } } @@ -2057,10 +2092,17 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t qstart = 0, qend = 0; getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); - pos += (qend - qstart + 1) * step; - - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart]; + if(qend >= qstart) { + // copy qend - qstart + 1 rows from file + numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); + int32_t num = qend - qstart + 1; + pos += num * step; + } else { + // nothing copy from file + pos += step; + } + + cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? keyFile[qend] : keyFile[qstart]; cur->lastKey = cur->win.ekey + step; } } while (numOfRows < pQueryHandle->outputCapacity); @@ -2077,7 +2119,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* !ASCENDING_TRAVERSE(pQueryHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { - cur->win.skey = tsArray[pos]; + cur->win.skey = keyFile[pos]; } int32_t start = -1, end = -1; @@ -2086,7 +2128,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; + cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? keyFile[end] : keyFile[start]; cur->lastKey = cur->win.ekey + step; cur->mixBlock = true; } @@ -2303,10 +2345,10 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO while (numOfTotal < cnt) { int32_t pos = pTree->pNode[0].index; - int32_t index = sup.blockIndexArray[pos]++; + int32_t tsd_index = sup.blockIndexArray[pos]++; STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; - pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; + pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[tsd_index]; // set data block index overflow, in order to disable the offset comparator if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { @@ -2855,6 +2897,9 @@ static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { } int32_t i = 0, j = 0; + + // lock pTable->lastCols[i] as it would be released when schema update(tsdbUpdateLastColSchema) + TSDB_RLOCK_TABLE(pTable); while(i < tgNumOfCols && j < numOfCols) { pColInfo = taosArrayGet(pQueryHandle->pColumns, i); if (pTable->lastCols[j].colId < pColInfo->info.colId) { @@ -2941,6 +2986,7 @@ static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { i++; j++; } + TSDB_RUNLOCK_TABLE(pTable); // leave the real ts column as the last row, because last function only (not stable) use the last row as res if (priKey != TSKEY_INITIAL_VAL) { @@ -2985,7 +3031,7 @@ static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); assert(numOfTables > 0); - int64_t stime = taosGetTimestampUs(); + int64_t tsd_stime = taosGetTimestampUs(); while(pQueryHandle->activeIndex < numOfTables) { if (loadBlockOfActiveTable(pQueryHandle)) { @@ -3003,7 +3049,7 @@ static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { terrno = TSDB_CODE_SUCCESS; - int64_t elapsedTime = taosGetTimestampUs() - stime; + int64_t elapsedTime = taosGetTimestampUs() - tsd_stime; pQueryHandle->cost.checkForNextTime += elapsedTime; } @@ -3013,14 +3059,17 @@ static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { // handle data in cache situation bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + if (pQueryHandle == NULL) { + return false; + } if (emptyQueryTimewindow(pQueryHandle)) { tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); return false; } - int64_t stime = taosGetTimestampUs(); - int64_t elapsedTime = stime; + int64_t tsd_stime = taosGetTimestampUs(); + int64_t elapsedTime = tsd_stime; // TODO refactor: remove "type" if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { @@ -3047,7 +3096,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { } if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - tsd_stime); return exists; } @@ -3059,7 +3108,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { bool ret = doHasDataInBuffer(pQueryHandle); terrno = TSDB_CODE_SUCCESS; - elapsedTime = taosGetTimestampUs() - stime; + elapsedTime = taosGetTimestampUs() - tsd_stime; pQueryHandle->cost.checkForNextTime += elapsedTime; return ret; } @@ -3132,6 +3181,9 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qId, pMemRef); tfree(cond.colList); + if (pSecQueryHandle == NULL) { + goto out_of_memory; + } // current table, only one table STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); @@ -3272,7 +3324,9 @@ int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { int32_t code = 0; - if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ + STsdbRepo* pRepo = pQueryHandle->pTsdb; + + if (pRepo && CACHE_LAST_NULL_COLUMN(&(pRepo->config))) { pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; } @@ -3403,7 +3457,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta return TSDB_CODE_SUCCESS; } - int64_t stime = taosGetTimestampUs(); + int64_t tsd_stime = taosGetTimestampUs(); if (tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock) < 0) { return terrno; } @@ -3433,7 +3487,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta } } - int64_t elapsed = taosGetTimestampUs() - stime; + int64_t elapsed = taosGetTimestampUs() - tsd_stime; pHandle->cost.statisInfoLoadTime += elapsed; *pBlockStatis = pHandle->statis; @@ -3695,6 +3749,9 @@ static bool tableFilterFp(const void* pNode, void* param) { return (val != NULL) && (!isNull(val, pInfo->sch.type)); } } else if (pInfo->optr == TSDB_RELATION_IN) { + if(val == NULL) { + return false; + } int type = pInfo->sch.type; if (type == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_TIMESTAMP) { int64_t v; diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index ef304d2fcbcb3a823e2c8253ca578de551499151..7ba87691cc7b926d558d0feed73dc4d6263adab2 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -15,7 +15,9 @@ TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic ${VAR_TSZ} ) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - ADD_SUBDIRECTORY(tests) + IF (NOT TD_NINGSI_60) + ADD_SUBDIRECTORY(tests) + ENDIF () FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index e41b544d00e55f7eece904c5957ef9c06063e6c3..40069d7d273caa14ce3b80467b25d68ea476fb75 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -33,6 +33,7 @@ extern "C" { #endif typedef void (*__cache_free_fn_t)(void*); +typedef void (*__cache_trav_fn_t)(void*, void*); typedef struct SCacheStatis { int64_t missCount; @@ -176,7 +177,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj); * @param fp * @return */ -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_free_fn_t fp); +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1); /** * stop background refresh worker thread diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index d6a1cb113a43874ee77b096e75959e02e4aa6747..e2165ec1a32c3b47a47a4b22243d1ac178d58876 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 120 +#define TSDB_CFG_MAX_NUM 122 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 diff --git a/src/util/inc/tthread.h b/src/util/inc/tthread.h new file mode 100644 index 0000000000000000000000000000000000000000..7443ad706dcbef529d857fe823cddd0cc1efbdd3 --- /dev/null +++ b/src/util/inc/tthread.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TTHREAD_H +#define TDENGINE_TTHREAD_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "taosdef.h" + +// create new thread +pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param); +// destory thread +bool taosDestoryThread(pthread_t* pthread); +// thread running return true +bool taosThreadRunning(pthread_t* pthread); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTHREAD_H diff --git a/src/util/src/hash.c b/src/util/src/hash.c index b6696810c12371bb8269149d082ad6890ddc4dd4..cda5f8a6625a34fe7c548027ade95c2079bdcad4 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -209,7 +209,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { if (!pHashObj) { return 0; } - return (int32_t)atomic_load_64(&pHashObj->size); + return (int32_t)atomic_load_64((volatile uint64_t *)&pHashObj->size); } static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 54b7e00eb7dd6f31ac8c8e6afa89790846abac5b..352cd3c05e4d588900b676b605964e068c4ed191 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -230,7 +230,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const { int32_t parent; int32_t child; - char *buf; + char *buf = NULL; if (base && size > 0 && compar) { parent = start; diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 2d6c513cb57ce1d524a1fb69df68702e624ede7b..2af495be8139e5358da61936ad3e5710894ad6d7 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -179,15 +179,15 @@ void* taosArrayPop(SArray* pArray) { return TARRAY_GET_ELEM(pArray, pArray->size); } -void* taosArrayGet(const SArray* pArray, size_t index) { - assert(index < pArray->size); - return TARRAY_GET_ELEM(pArray, index); +void* taosArrayGet(const SArray* pArray, size_t utl_index) { + assert(utl_index < pArray->size); + return TARRAY_GET_ELEM(pArray, utl_index); } -void* taosArrayGetP(const SArray* pArray, size_t index) { - assert(index < pArray->size); +void* taosArrayGetP(const SArray* pArray, size_t utl_index) { + assert(utl_index < pArray->size); - void* d = TARRAY_GET_ELEM(pArray, index); + void* d = TARRAY_GET_ELEM(pArray, utl_index); return *(void**)d; } @@ -203,12 +203,12 @@ void taosArraySetSize(SArray* pArray, size_t size) { pArray->size = size; } -void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { +void* taosArrayInsert(SArray* pArray, size_t utl_index, void* pData) { if (pArray == NULL || pData == NULL) { return NULL; } - if (index >= pArray->size) { + if (utl_index >= pArray->size) { return taosArrayPush(pArray, pData); } @@ -220,9 +220,9 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { } } - void* dst = TARRAY_GET_ELEM(pArray, index); + void* dst = TARRAY_GET_ELEM(pArray, utl_index); - int32_t remain = (int32_t)(pArray->size - index); + int32_t remain = (int32_t)(pArray->size - utl_index); memmove((char*)dst + pArray->elemSize, (char*)dst, pArray->elemSize * remain); memcpy(dst, pData, pArray->elemSize); @@ -231,21 +231,21 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { return dst; } -void taosArraySet(SArray* pArray, size_t index, void* pData) { - assert(index < pArray->size); - memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize); +void taosArraySet(SArray* pArray, size_t utl_index, void* pData) { + assert(utl_index < pArray->size); + memcpy(TARRAY_GET_ELEM(pArray, utl_index), pData, pArray->elemSize); } -void taosArrayRemove(SArray* pArray, size_t index) { - assert(index < pArray->size); +void taosArrayRemove(SArray* pArray, size_t utl_index) { + assert(utl_index < pArray->size); - if (index == pArray->size - 1) { + if (utl_index == pArray->size - 1) { taosArrayPop(pArray); return; } - size_t remain = pArray->size - index - 1; - memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + 1) * pArray->elemSize, remain * pArray->elemSize); + size_t remain = pArray->size - utl_index - 1; + memmove((char*)pArray->pData + utl_index * pArray->elemSize, (char*)pArray->pData + (utl_index + 1) * pArray->elemSize, remain * pArray->elemSize); pArray->size -= 1; } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 776a0ad9c83c0f133a23f4f2832acd473ed6ccec..f49dd16c3943040bcc9b58000a87845ea1c87b13 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -505,7 +505,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { typedef struct SHashTravSupp { SCacheObj* pCacheObj; int64_t time; - __cache_free_fn_t fp; + __cache_trav_fn_t fp; + void* param1; } SHashTravSupp; static bool travHashTableEmptyFn(void* param, void* data) { @@ -615,9 +616,9 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { return; } - const char* stat[] = {"false", "true"}; + const char* utl_stat[] = {"false", "true"}; uDebug("cache:%s start to cleanup trashcan, numOfElem in trashcan:%d, free:%s", pCacheObj->name, - pCacheObj->numOfElemsInTrash, (force? stat[1]:stat[0])); + pCacheObj->numOfElemsInTrash, (force? utl_stat[1]:utl_stat[0])); STrashElem *pElem = pCacheObj->pTrash; while (pElem) { @@ -667,17 +668,17 @@ bool travHashTableFn(void* param, void* data) { } if (ps->fp) { - (ps->fp)(pNode->data); + (ps->fp)(pNode->data, ps->param1); } // do not remove element in hash table return true; } -static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t fp) { +static void doCacheRefresh(SCacheObj* pCacheObj, int64_t utl_time, __cache_trav_fn_t fp, void* param1) { assert(pCacheObj != NULL); - SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time}; + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = utl_time, .param1 = param1}; taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); } @@ -748,7 +749,7 @@ void* taosCacheTimedRefresh(void *handle) { // refresh data in hash table if (elemInHash > 0) { int64_t now = taosGetTimestampMs(); - doCacheRefresh(pCacheObj, now, NULL); + doCacheRefresh(pCacheObj, now, NULL, NULL); } taosTrashcanEmpty(pCacheObj, false); @@ -766,13 +767,13 @@ void* taosCacheTimedRefresh(void *handle) { return NULL; } -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_free_fn_t fp) { +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1) { if (pCacheObj == NULL) { return; } int64_t now = taosGetTimestampMs(); - doCacheRefresh(pCacheObj, now, fp); + doCacheRefresh(pCacheObj, now, fp, param1); } void taosStopCacheRefreshWorker(void) { diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 47cc75131802fce5c72e7fdd3ae6675d34917a8c..3a5f3e368b7fec8a67a1fa0363c59ba820d86248 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -263,6 +263,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat if (j <= size) { if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; } + if (c == '\\' && patterStr[i] == '%' && c1 == '%') { i++; continue; } if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { continue; } @@ -294,9 +295,9 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c return TSDB_PATTERN_MATCH; } - wchar_t accept[3] = {towupper(c), towlower(c), 0}; + wchar_t utl_accept[3] = {towupper(c), towlower(c), 0}; while (1) { - size_t n = wcscspn(str, accept); + size_t n = wcscspn(str, utl_accept); str += n; if (str[0] == 0 || (n >= size)) { @@ -357,14 +358,20 @@ int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; + size_t size = varDataLen(pLeft)/TSDB_NCHAR_SIZE; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); + wchar_t *str = calloc(size + 1, sizeof(wchar_t)); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + memcpy(str, varDataVal(pLeft), size * sizeof(wchar_t)); + + int32_t ret = WCSPatternMatch(pattern, str, size, &pInfo); - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); free(pattern); + free(str); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 48bba75926415752cfd777242a55ef71c5c96c2c..66150c46fb22a7225444b4e13b2d35fc636ea15b 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -346,7 +346,7 @@ int tsCompressBoolImp(const char *const input, const int nelements, char *const /* t = (~((( uint8_t)1) << (7-i%BITS_PER_BYTE))); */ output[pos] |= t; } else { - uError("Invalid compress bool value:%d", output[pos]); + uError("Invalid compress bool value:%d", input[i]); return -1; } } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 131d05155283375fefcb5c8b0851c8b2236e83af..ff076522cade7b096cf01085e73f7509d2cf9ba8 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -330,25 +330,34 @@ void taosReadGlobalLogCfg() { } strcpy(configDir, full_path.we_wordv[0]); } else { - #ifdef _TD_POWER_ +#ifdef _TD_POWER_ printf("configDir:%s not there, use default value: /etc/power", configDir); strcpy(configDir, "/etc/power"); - #elif (_TD_TQ_ == true) +#elif (_TD_TQ_ == true) printf("configDir:%s not there, use default value: /etc/tq", configDir); strcpy(configDir, "/etc/tq"); - #elif (_TD_PRO_ == true) +#elif (_TD_PRO_ == true) printf("configDir:%s not there, use default value: /etc/ProDB", configDir); strcpy(configDir, "/etc/ProDB"); - #else +#elif (_TD_KH_ == true) + printf("configDir:%s not there, use default value: /etc/kinghistorian", configDir); + strcpy(configDir, "/etc/kinghistorian"); +#elif (_TD_JH_ == true) + printf("configDir:%s not there, use default value: /etc/jh_taos", configDir); + strcpy(configDir, "/etc/jh_taos"); +#else printf("configDir:%s not there, use default value: /etc/taos", configDir); strcpy(configDir, "/etc/taos"); - #endif +#endif } wordfree(&full_path); taosReadLogOption("logDir", tsLogDir); - +#ifdef _TD_KH_ + sprintf(fileName, "%s/kinghistorian.cfg", configDir); +#else sprintf(fileName, "%s/taos.cfg", configDir); +#endif fp = fopen(fileName, "r"); if (fp == NULL) { printf("\nconfig file:%s not found, all variables are set to default\n", fileName); @@ -392,8 +401,11 @@ bool taosReadGlobalCfg() { char * line, *option, *value, *value2, *value3; int olen, vlen, vlen2, vlen3; char fileName[PATH_MAX] = {0}; - +#ifdef _TD_KH_ + sprintf(fileName, "%s/kinghistorian.cfg", configDir); +#else sprintf(fileName, "%s/taos.cfg", configDir); +#endif FILE* fp = fopen(fileName, "r"); if (fp == NULL) { struct stat s; @@ -455,7 +467,7 @@ bool taosReadGlobalCfg() { } void taosPrintGlobalCfg() { - uInfo(" taos config & system info:"); + uInfo(" config & system info:"); uInfo("=================================="); for (int i = 0; i < tsGlobalConfigNum; ++i) { @@ -541,7 +553,7 @@ static void taosDumpCfg(SGlobalCfg *cfg) { } void taosDumpGlobalCfg() { - printf("taos global config:\n"); + printf(" global config:\n"); printf("==================================\n"); for (int i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; @@ -552,7 +564,7 @@ void taosDumpGlobalCfg() { taosDumpCfg(cfg); } - printf("\ntaos local config:\n"); + printf("\n local config:\n"); printf("==================================\n"); for (int i = 0; i < tsGlobalConfigNum; ++i) { diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 0b9a76004b3e445abe1bbffe593d8061803d1b1b..d1ecc2188bec5c4a5b1af36a67cf177dd6d935ec 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -112,9 +112,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too lon TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_RES_TOO_MANY, "Result set too large to be output") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 0d335ca2664ffee75a79144b97181a5b625df66d..232d10a7d07594c9c62cd13767c320da27af2a73 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -87,6 +87,10 @@ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/tq"; #elif (_TD_PRO_ == true) char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/ProDB"; +#elif (_TD_KH_ == true) +char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/kinghistorian"; +#elif (_TD_JH_ == true) +char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/jh_taos"; #else char tsLogDir[PATH_MAX] = "/var/log/taos"; #endif @@ -566,7 +570,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) int32_t end = 0; int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; + char tmpBuf[60] = {0}; int32_t tmpBufLen = 0; if (tLogBuff == NULL || tLogBuff->stop) return -1; diff --git a/src/util/src/tmempool.c b/src/util/src/tmempool.c index 678c965eb1a7315977616778c0e4b39ceb4c7525..68b1893d2a84691a3780087ee0a1cc65472c8877 100644 --- a/src/util/src/tmempool.c +++ b/src/util/src/tmempool.c @@ -89,19 +89,19 @@ char *taosMemPoolMalloc(mpool_h handle) { } void taosMemPoolFree(mpool_h handle, char *pMem) { - int index; + int utl_index; pool_t *pool_p = (pool_t *)handle; if (pMem == NULL) return; - index = (int)(pMem - pool_p->pool) % pool_p->blockSize; - if (index != 0) { + utl_index = (int)(pMem - pool_p->pool) % pool_p->blockSize; + if (utl_index != 0) { uError("invalid free address:%p\n", pMem); return; } - index = (int)((pMem - pool_p->pool) / pool_p->blockSize); - if (index < 0 || index >= pool_p->numOfBlock) { + utl_index = (int)((pMem - pool_p->pool) / pool_p->blockSize); + if (utl_index < 0 || utl_index >= pool_p->numOfBlock) { uError("mempool: error, invalid address:%p\n", pMem); return; } @@ -110,7 +110,7 @@ void taosMemPoolFree(mpool_h handle, char *pMem) { pthread_mutex_lock(&pool_p->mutex); - pool_p->freeList[(pool_p->first + pool_p->numOfFree) % pool_p->numOfBlock] = index; + pool_p->freeList[(pool_p->first + pool_p->numOfFree) % pool_p->numOfBlock] = utl_index; pool_p->numOfFree++; pthread_mutex_unlock(&pool_p->mutex); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index ed91f95180d8ce4a03e0cce02e64b324bcee59d9..bef75c8100f302b8409b56dbf7fda2c3d9427f22 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -314,27 +314,27 @@ static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort } void *taosNetInitRpc(char *secretEncrypt, char spi) { - SRpcInit rpcInit; + SRpcInit utl_rpcInit; void * pRpcConn = NULL; char user[] = "nettestinternal"; char pass[] = "nettestinternal"; taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "NT"; - rpcInit.numOfThreads = 1; // every DB connection has only one thread - rpcInit.cfp = NULL; - rpcInit.sessions = 16; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = user; - rpcInit.idleTime = 2000; - rpcInit.ckey = "key"; - rpcInit.spi = spi; - rpcInit.secret = secretEncrypt; - - pRpcConn = rpcOpen(&rpcInit); + memset(&utl_rpcInit, 0, sizeof(utl_rpcInit)); + utl_rpcInit.localPort = 0; + utl_rpcInit.label = "NT"; + utl_rpcInit.numOfThreads = 1; // every DB connection has only one thread + utl_rpcInit.cfp = NULL; + utl_rpcInit.sessions = 16; + utl_rpcInit.connType = TAOS_CONN_CLIENT; + utl_rpcInit.user = user; + utl_rpcInit.idleTime = 2000; + utl_rpcInit.ckey = "key"; + utl_rpcInit.spi = spi; + utl_rpcInit.secret = secretEncrypt; + + pRpcConn = rpcOpen(&utl_rpcInit); return pRpcConn; } diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 33323889c68162219b3c6faf886ac29b2a975ffa..d8178221083a26165cbfc91b03cbe5bef67e2cdf 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -54,7 +54,7 @@ static void taosLockList(int64_t *lockedBy); static void taosUnlockList(int64_t *lockedBy); static void taosIncRsetCount(SRefSet *pSet); static void taosDecRsetCount(SRefSet *pSet); -static int taosDecRefCount(int rsetId, int64_t rid, int remove); +static int taosDecRefCount(int rsetId, int64_t rid, int utl_remove); int taosOpenRef(int max, void (*fp)(void *)) { @@ -389,7 +389,7 @@ int taosListRef() { return num; } -static int taosDecRefCount(int rsetId, int64_t rid, int remove) { +static int taosDecRefCount(int rsetId, int64_t rid, int utl_remove) { int hash; SRefSet *pSet; SRefNode *pNode; @@ -428,7 +428,7 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (pNode) { pNode->count--; - if (remove) pNode->removed = 1; + if (utl_remove) pNode->removed = 1; if (pNode->count <= 0) { if (pNode->prev) { diff --git a/src/util/src/tthread.c b/src/util/src/tthread.c new file mode 100644 index 0000000000000000000000000000000000000000..043b2de2f241297d209041294428dde2c55e974e --- /dev/null +++ b/src/util/src/tthread.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tthread.h" +#include "tglobal.h" +#include "taosdef.h" +#include "tutil.h" +#include "tulog.h" +#include "taoserror.h" + +// create new thread +pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { + pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int32_t ret = pthread_create(pthread, &thattr, __start_routine, param); + pthread_attr_destroy(&thattr); + + if (ret != 0) { + free(pthread); + return NULL; + } + return pthread; +} + +// destory thread +bool taosDestoryThread(pthread_t* pthread) { + if(pthread == NULL) return false; + if(taosThreadRunning(pthread)) { + pthread_cancel(*pthread); + pthread_join(*pthread, NULL); + } + + free(pthread); + return true; +} + +// thread running return true +bool taosThreadRunning(pthread_t* pthread) { + if(pthread == NULL) return false; + int ret = pthread_kill(*pthread, 0); + if(ret == ESRCH) + return false; + if(ret == EINVAL) + return false; + // alive + return true; +} diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index ae10dd265a5f94d2a2f3a30888b917b617dec84e..7d7fa4b8c877bb35c9ccbaaa378a5f04f9560bfa 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -64,7 +64,7 @@ int32_t strRmquote(char *z, int32_t len){ int32_t j = 0; for (uint32_t k = 1; k < len - 1; ++k) { if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { - if (z[k] == '\\' && z[k + 1] == '_') { + if ((z[k] == '\\' && z[k + 1] == '_') || (z[k] == '\\' && z[k + 1] == '%')) { //match '_' self } else { z[j] = z[k + 1]; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f826c1aecd336a0eedeb3f02df0a7acc61895bb2..60c60cf531c815023fb18eafcc79d7c41b5cae4f 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -519,9 +519,9 @@ void vnodeCleanUp(SVnodeObj *pVnode) { // stop replication module if (pVnode->sync > 0) { - int64_t sync = pVnode->sync; + int64_t vnd_sync = pVnode->sync; pVnode->sync = -1; - syncStop(sync); + syncStop(vnd_sync); } vDebug("vgId:%d, vnode is cleaned, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); @@ -560,5 +560,10 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { return vnodeSaveVersion(pVnode); } + // timer thread callback + if(status == TSDB_STATUS_COMMIT_NOBLOCK) { + qSolveCommitNoBlock(pVnode->tsdb, pVnode->qMgmt); + } + return 0; } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 2bdfd2ead3a31d8c2cba94d93239de965d2e07dc..1929479deb36bfc76b23b21b8e9c0f509328b2ba 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -22,7 +22,7 @@ #include "vnodeMain.h" #include "vnodeStatus.h" -uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) { +uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *vnd_index, uint32_t eindex, int64_t *size, uint64_t *fver) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while get file info", vgId); @@ -30,7 +30,7 @@ uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t ei } *fver = pVnode->fversion; - uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); + uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, vnd_index, eindex, size); vnodeRelease(pVnode); return ret; diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 505728fbe4c4a6fbc126aa18ff6db93a28388173..7d6505cd195ff9c87cf26aba3ea7fd0a668c2a37 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -113,17 +113,17 @@ int main(int argc, char *argv[]) { printf("%d wal files are written\n", total); - int64_t index = 0; + int64_t wal_index = 0; char name[256]; while (1) { - int code = walGetWalFile(pWal, name, &index); + int code = walGetWalFile(pWal, name, &wal_index); if (code == -1) { - printf("failed to get wal file, index:%" PRId64 "\n", index); + printf("failed to get wal file, index:%" PRId64 "\n", wal_index); break; } - printf("index:%" PRId64 " wal:%s\n", index, name); + printf("index:%" PRId64 " wal:%s\n", wal_index, name); if (code == 0) break; } diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 55d962888871c2ba175daef85f1084a1e28a0da1..ada75d690f1a01edf6997e9dee7d45bbb56f5e65 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -72,8 +72,8 @@ int main(int argc, char *argv[]) { taos_close(taos); taos_cleanup(); } -void Test(TAOS *taos, char *qstr, int index) { - printf("==================test at %d\n================================", index); +void Test(TAOS *taos, char *qstr, int tst_index) { + printf("==================test at %d\n================================", tst_index); queryDB(taos, "drop database if exists demo"); queryDB(taos, "create database demo"); TAOS_RES *result; diff --git a/tests/examples/c/stream.c b/tests/examples/c/stream.c index e26d6588a1382a1f7133f84bd623aa05304060a9..28a3eaddc48ea43044325ac61bc98c43e2f83a10 100644 --- a/tests/examples/c/stream.c +++ b/tests/examples/c/stream.c @@ -155,12 +155,12 @@ void *insert_rows(void *sarg) { // insert data int64_t begin = (int64_t)time(NULL); - int index = 0; + int tst_index = 0; while (1) { if (g_thread_exit_flag) break; - index++; - sprintf(command, "insert into %s values (%ld, %d)", winfo->tbl_name, (begin + index) * 1000, index); + tst_index++; + sprintf(command, "insert into %s values (%ld, %d)", winfo->tbl_name, (begin + tst_index) * 1000, tst_index); if (taos_query(taos, command)) { printf("failed to insert row [%s], reason:%s\n", command, taos_errstr(taos)); } diff --git a/tests/examples/lua/lua51/lua_connector51.c b/tests/examples/lua/lua51/lua_connector51.c index fe2152945dc1915dca5de31458a8cbb2f007f4f2..a5f6f2ea40695e28dcb6d49f578a0f3b978b6f35 100644 --- a/tests/examples/lua/lua51/lua_connector51.c +++ b/tests/examples/lua/lua51/lua_connector51.c @@ -216,7 +216,7 @@ static int l_async_query(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); TAOS * taos = (TAOS*)lua_topointer(L,1); const char * sqlstr = lua_tostring(L,2); - // int stime = luaL_checknumber(L,3); + // int test_stime = luaL_checknumber(L,3); lua_newtable(L); int table_index = lua_gettop(L); @@ -301,7 +301,7 @@ static int l_open_stream(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); TAOS * taos = (TAOS*)lua_topointer(L,1); const char * sqlstr = lua_tostring(L,2); - int stime = luaL_checknumber(L,3); + int test_stime = luaL_checknumber(L,3); lua_newtable(L); int table_index = lua_gettop(L); @@ -310,7 +310,7 @@ static int l_open_stream(lua_State *L){ p->state = L; p->callback=r; // printf("r:%d, L:%d\n",r,L); - void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL); + void * s = taos_open_stream(taos,sqlstr,stream_cb,test_stime,p,NULL); if (s == NULL) { printf("failed to open stream, reason:%s\n", taos_errstr(taos)); free(p); diff --git a/tests/examples/lua/lua_connector.c b/tests/examples/lua/lua_connector.c index 8c2ea3e9e83237fc8ed9ebce687f5131352e4d14..b4097be36a3643c1c3c95b0ae07b3792b82bb5ee 100644 --- a/tests/examples/lua/lua_connector.c +++ b/tests/examples/lua/lua_connector.c @@ -216,7 +216,7 @@ static int l_async_query(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); TAOS * taos = (TAOS*)lua_topointer(L,1); const char * sqlstr = lua_tostring(L,2); - // int stime = luaL_checknumber(L,3); + // int test_stime = luaL_checknumber(L,3); lua_newtable(L); int table_index = lua_gettop(L); @@ -301,7 +301,7 @@ static int l_open_stream(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); TAOS * taos = (TAOS*)lua_topointer(L,1); const char * sqlstr = lua_tostring(L,2); - int stime = luaL_checknumber(L,3); + int test_stime = luaL_checknumber(L,3); lua_newtable(L); int table_index = lua_gettop(L); @@ -310,7 +310,7 @@ static int l_open_stream(lua_State *L){ p->state = L; p->callback=r; // printf("r:%d, L:%d\n",r,L); - void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL); + void * s = taos_open_stream(taos,sqlstr,stream_cb,test_stime,p,NULL); if (s == NULL) { printf("failed to open stream, reason:%s\n", taos_errstr(taos)); free(p); diff --git a/tests/examples/python/taosdemo/taosdemo.py b/tests/examples/python/taosdemo/taosdemo.py index d55023bdbf119544a788aa6246c9d63dbf024872..4aaf00157c5fe5bbeec27b001f663a94c1d89439 100755 --- a/tests/examples/python/taosdemo/taosdemo.py +++ b/tests/examples/python/taosdemo/taosdemo.py @@ -21,78 +21,91 @@ import json import random import time import datetime +import multiprocessing from multiprocessing import Manager, Pool, Lock from multipledispatch import dispatch from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED @dispatch(str, str) -def v_print(msg: str, arg: str): +def v_print(msg, arg): + # type: (str, str) -> None if verbose: print(msg % arg) @dispatch(str, str, str) -def v_print(msg: str, arg1: str, arg2: str): +def v_print(msg, arg1, arg2): + # type: (str, str, str) -> None if verbose: print(msg % (arg1, arg2)) @dispatch(str, str, str, str) -def v_print(msg: str, arg1: str, arg2: str, arg3: str): +def v_print(msg, arg1, arg2, arg3): + # type: (str, str, str, str) -> None if verbose: print(msg % (arg1, arg2, arg3)) @dispatch(str, str, str, str, str) -def v_print(msg: str, arg1: str, arg2: str, arg3: str, arg4: str): +def v_print(msg, arg1, arg2, arg3, arg4): + # type: (str, str, str, str, str) -> None if verbose: print(msg % (arg1, arg2, arg3, arg4)) @dispatch(str, int) -def v_print(msg: str, arg: int): +def v_print(msg, arg): + # type: (str, int) -> None if verbose: print(msg % int(arg)) @dispatch(str, int, str) -def v_print(msg: str, arg1: int, arg2: str): +def v_print(msg, arg1, arg2): + # type: (str, int, str) -> None if verbose: print(msg % (int(arg1), str(arg2))) @dispatch(str, str, int) -def v_print(msg: str, arg1: str, arg2: int): +def v_print(msg, arg1, arg2): + # type: (str, str, int) -> None if verbose: print(msg % (arg1, int(arg2))) @dispatch(str, int, int) -def v_print(msg: str, arg1: int, arg2: int): +def v_print(msg, arg1, arg2): + # type: (str, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2))) @dispatch(str, int, int, str) -def v_print(msg: str, arg1: int, arg2: int, arg3: str): +def v_print(msg, arg1, arg2, arg3): + # type: (str, int, int, str) -> None if verbose: print(msg % (int(arg1), int(arg2), str(arg3))) @dispatch(str, int, int, int) -def v_print(msg: str, arg1: int, arg2: int, arg3: int): +def v_print(msg, arg1, arg2, arg3): + # type: (str, int, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2), int(arg3))) @dispatch(str, int, int, int, int) -def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): +def v_print(msg, arg1, arg2, arg3, arg4): + # type: (str, int, int, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) -def restful_execute(host: str, port: int, user: str, password: str, cmd: str): +def restful_execute(host, port, user, password, cmd): + # type: (str, int, str, str, str) -> None url = "http://%s:%d/rest/sql" % (host, restPort) v_print("restful_execute - cmd: %s", cmd) @@ -112,7 +125,8 @@ def restful_execute(host: str, port: int, user: str, password: str, cmd: str): print("resp: %s" % json.dumps(resp.json())) -def query_func(process: int, thread: int, cmd: str): +def query_func(process, thread, cmd): + # type: (int, int, str) -> None v_print("%d process %d thread cmd: %s", process, thread, cmd) if oneMoreHost != "NotSupported" and random.randint( @@ -133,7 +147,8 @@ def query_func(process: int, thread: int, cmd: str): host, port, user, password, cmd) -def query_data_process(cmd: str): +def query_data_process(cmd): + # type: (str) -> None # establish connection if native if native: v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) @@ -256,7 +271,8 @@ def drop_databases(): (dbName, i)) -def insert_func(process: int, thread: int): +def insert_func(process, thread): + # type: (int, int) -> None v_print("%d process %d thread, insert_func ", process, thread) # generate uuid @@ -374,7 +390,8 @@ def create_tb(): (tbName, j)) -def insert_data_process(lock, i: int, begin: int, end: int): +def insert_data_process(lock, i, begin, end): + # type: (multiprocessing._LockType, int, int, int) -> None lock.acquire() tasks = end - begin v_print("insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks) @@ -675,7 +692,10 @@ if __name__ == "__main__": printConfig() if not skipPrompt: - input("Press any key to continue..") + try: + input("Press any key to continue..") + except SyntaxError: + pass # establish connection first if native if native: diff --git a/tests/pytest/compress/compressChangeVersion.py b/tests/pytest/compress/compressChangeVersion.py new file mode 100644 index 0000000000000000000000000000000000000000..b7b9ebe6b35dc4729e0dcae705ac7d93c73010e7 --- /dev/null +++ b/tests/pytest/compress/compressChangeVersion.py @@ -0,0 +1,109 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading +import subprocess +from random import choice + + +class TwoClients: + def initConnection(self): + self.host = "chenhaoran01" + self.user = "root" + self.password = "taosdata" + self.config = "/home/chr/cfg/single/" + self.port =6030 + self.rowNum = 10 + self.ts = 1537146000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + walFilePath = "/var/lib/taos/mnode_bak/wal/" + + # new taos client + conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn1) + cur1 = conn1.cursor() + tdSql.init(cur1, True) + + # create backgroud db and tb + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f compress/insertDataDb1.json -y " % binPath) + # create foreground db and tb + tdSql.execute("drop database if exists foredb") + tdSql.execute("create database foredb") + tdSql.execute("use foredb") + print("123test") + tdSql.execute("create stable if not exists stb (ts timestamp, dataInt int, dataDouble double,dataStr nchar(200)) tags(loc nchar(50),t1 int)") + tdSql.execute("create table tb1 using stb tags('beijing1', 10)") + tdSql.execute("insert into tb1 values(1614218412000,8635,98.861,'qazwsxedcrfvtgbyhnujmikolp1')(1614218422000,8636,98.862,'qazwsxedcrfvtgbyhnujmikolp2')") + tdSql.execute("create table tb2 using stb tags('beijing2', 11)") + tdSql.execute("insert into tb2 values(1614218432000,8647,98.863,'qazwsxedcrfvtgbyhnujmikolp3')") + tdSql.execute("insert into tb2 values(1614218442000,8648,98.864,'qazwsxedcrfvtgbyhnujmikolp4')") + + + # check data correct + tdSql.execute("use db1") + tdSql.query("select count(tbname) from stb0") + tdSql.checkData(0, 0, 50000) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 5000000) + tdSql.execute("use foredb") + tdSql.query("select count (tbname) from stb") + tdSql.checkData(0, 0, 2) + tdSql.query("select count (*) from stb") + tdSql.checkData(0, 0, 4) + tdSql.query("select * from tb1 order by ts") + tdSql.checkData(0, 3, "qazwsxedcrfvtgbyhnujmikolp1") + tdSql.query("select * from tb2 order by ts") + tdSql.checkData(1, 3, "qazwsxedcrfvtgbyhnujmikolp4") + + + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + # os.system("rm -rf compress/%s.sql" % testcaseFilename ) + +clients = TwoClients() +clients.initConnection() +# clients.getBuildPath() +clients.run() \ No newline at end of file diff --git a/tests/pytest/compress/insertDataDb1.json b/tests/pytest/compress/insertDataDb1.json new file mode 100644 index 0000000000000000000000000000000000000000..65cec71a65ff4ef3814bee4949def151c32945ee --- /dev/null +++ b/tests/pytest/compress/insertDataDb1.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 50000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"},{"type": "TINYINT"},{"type": "smallint"},{"type": "bool"},{"type": "bigint"},{"type": "float"},{"type": "double"}, {"type": "BINARY","len": 32}, {"type": "nchar","len": 32}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/fulltest.bat b/tests/pytest/fulltest.bat index ad3803b01fc3733d2c17a7706e3dc4dee4d97f04..535aafe2d4692f4dc85756256390502c5fd96efd 100644 --- a/tests/pytest/fulltest.bat +++ b/tests/pytest/fulltest.bat @@ -18,4 +18,4 @@ python .\test.py -f query\filterAllIntTypes.py python .\test.py -f query\filterFloatAndDouble.py python .\test.py -f query\filterOtherTypes.py python .\test.py -f query\querySort.py -python .\test.py -f query\queryJoin.py \ No newline at end of file +python .\test.py -f query\queryJoin.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 496a59c4bd2f2694d0a7c416ec1ee9da96c98bd1..d712e04edf860ddef5b607bdced440ce3ec9def3 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -29,6 +29,7 @@ python3 ./test.py -f insert/in_function.py python3 ./test.py -f insert/modify_column.py python3 ./test.py -f insert/line_insert.py python3 ./test.py -f insert/specialSql.py +python3 ./test.py -f insert/timestamp.py #table python3 ./test.py -f table/alter_wal0.py @@ -42,6 +43,7 @@ python3 ./test.py -f table/alter_column.py python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py +python3 ./test.py -f table/create_db_from_normal_db.py #stable python3 ./test.py -f stable/insert.py @@ -264,10 +266,12 @@ python3 ./test.py -f query/queryStateWindow.py # python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py python3 ./test.py -f query/nestquery_last_row.py python3 ./test.py -f query/nestedQuery/nestedQuery.py +python3 ./test.py -f query/nestedQuery/nestedQuery_datacheck.py python3 ./test.py -f query/queryCnameDisplay.py python3 ./test.py -f query/operator_cost.py python3 test.py -f query/nestedQuery/queryWithSpread.py python3 ./test.py -f query/bug6586.py +# python3 ./test.py -f query/bug5903.py #stream python3 ./test.py -f stream/metric_1.py @@ -357,7 +361,7 @@ python3 ./test.py -f query/last_row_cache.py python3 ./test.py -f account/account_create.py python3 ./test.py -f alter/alter_table.py python3 ./test.py -f query/queryGroupbySort.py -python3 ./test.py -f functions/queryTestCases.py +#python3 ./test.py -f functions/queryTestCases.py python3 ./test.py -f functions/function_stateWindow.py python3 ./test.py -f functions/function_derivative.py python3 ./test.py -f functions/function_irate.py @@ -398,6 +402,7 @@ python3 ./test.py -f query/queryWildcardLength.py python3 ./test.py -f query/queryTbnameUpperLower.py python3 ./test.py -f query/queryGroupTbname.py python3 ./test.py -f insert/verifyMemToDiskCrash.py +python3 ./test.py -f functions/variable_httpDbNameMandatory.py #======================p4-end=============== diff --git a/tests/pytest/functions/function_count.py b/tests/pytest/functions/function_count.py index 9812473d6496a94447734f1f3507ac806e392294..a05b2b072090fb03a86015616ea138fb9df9ae75 100644 --- a/tests/pytest/functions/function_count.py +++ b/tests/pytest/functions/function_count.py @@ -74,6 +74,25 @@ class TDTestCase: tdSql.execute("alter table test add column col10 int") tdSql.query("select count(col10) from test") tdSql.checkRows(0) + + #TS-653 forbidden count(tbname) to mix up with agg, proj etc. + tdSql.error("select count(*),count(tbname) from test") + tdSql.error("select col11,count(tbname) from test;") + tdSql.error("select count(tbname),count(*) from test") + tdSql.error("select count(tbname),col11 from test;") + func_list=['avg','count','twa','sum','stddev','leastsquares','min', + 'max','first','last','top','bottom','percentile','apercentile', + 'last_row','diff','spread' + ] + for j in func_list: + if j == 'leastsquares': + pick_func=j+'(col1,1,1)' + elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile': + pick_func=j+'(col1,1)' + else: + pick_func=j+'(col)' + tdSql.error("select {0},count(tbname) from test".format(pick_func)) + tdSql.error("select count(tbname),{0} from test".format(pick_func)) tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' , 1, 1, 1, 1, 1)") tdSql.query("select count(col10) from test") diff --git a/tests/pytest/functions/function_count_last_stab.py b/tests/pytest/functions/function_count_last_stab.py index 1d777c6bd314941036f542c7d0e9063e590fa7dd..cd0a9b17c2fd8c98544dca09f6a7008929225ece 100644 --- a/tests/pytest/functions/function_count_last_stab.py +++ b/tests/pytest/functions/function_count_last_stab.py @@ -42,22 +42,22 @@ class TDTestCase: % (self.ts + i, i + 1, 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select count(*),last(*) from stest group by col1") - tdSql.checkRows(10) - tdSql.checkData(0, 0, 1) - tdSql.checkData(1, 2, 2) - tdSql.checkData(1, 3, 1) + tdSql.checkRows(11) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 2, 2) + tdSql.checkData(2, 3, 1) tdSql.query("select count(*),last(*) from stest group by col2") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 10) - tdSql.checkData(0, 2, 10) - tdSql.checkData(0, 3, 1) + tdSql.checkRows(2) + tdSql.checkData(1, 0, 10) + tdSql.checkData(1, 2, 10) + tdSql.checkData(1, 3, 1) tdSql.query("select count(*),last(ts,stest.*) from stest group by col1") - tdSql.checkRows(10) - tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 2, "2018-09-17 09:00:00") - tdSql.checkData(1, 4, 1) + tdSql.checkRows(11) + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 2, "2018-09-17 09:00:00") + tdSql.checkData(2, 4, 1) diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index d4d8ab80a6b9587df900890ef18b8c4b1e3906bd..3b79726ed80c206338392cecb8f3d2adf4588e2a 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -68,9 +68,9 @@ class TDTestCase: tdSql.checkData(0, 0, "2018-09-17 09:00:10.000") tdSql.checkData(0, 1, "2018-09-17 09:00:10.000") tdSql.checkData(0, 3, "2018-09-17 09:00:10.000") - tdSql.checkData(1, 0, "2018-09-17 09:00:20.009") - tdSql.checkData(1, 1, "2018-09-17 09:00:20.009") - tdSql.checkData(1, 3, "2018-09-17 09:00:20.009") + tdSql.checkData(1, 0, "2018-09-17 09:00:20.000") + tdSql.checkData(1, 1, "2018-09-17 09:00:20.000") + tdSql.checkData(1, 3, "2018-09-17 09:00:20.000") tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname)") @@ -147,6 +147,10 @@ class TDTestCase: tdSql.error("select derivative(col, 10s, 1) from stb group by id") tdSql.error("select derivative(col, 999ms, 1) from stb group by id") tdSql.error("select derivative(col, 10s, 2) from stb group by id") + tdSql.error("select derivative(col, -106752999999999922222d, 0) from stb group by tbname"); #overflow error + tdSql.error("select derivative(col, 10y, 0) from stb group by tbname") #TD-10399, DB error: syntax error near '10y, 0) from stb group by tbname;' + tdSql.error("select derivative(col, -106752d, 0) from stb group by tbname") #TD-10398 overflow tips + tdSql.error("select derivative(col, 106751991168d, 0) from stb group by tbname") #TD-10398 overflow tips def run(self): tdSql.prepare() diff --git a/tests/pytest/functions/function_stddev.py b/tests/pytest/functions/function_stddev.py index 3ff2b82bf6b326ed4d07a5a51027c9e266c2fd72..b8b0b79739e310eac90833c8ab07289baa008538 100644 --- a/tests/pytest/functions/function_stddev.py +++ b/tests/pytest/functions/function_stddev.py @@ -123,8 +123,34 @@ class TDTestCase: tdSql.execute("insert into t1 values(now, 1, 'abc');") tdLog.info("select stddev(k) from t1 where b <> 'abc' interval(1s);") tdSql.query("select stddev(k) from t1 where b <> 'abc' interval(1s);") - - + + #add for TD-11459 + tdSql.execute("create table stdtable(ts timestamp, col1 int) tags(loc nchar(64))") + tdSql.execute("create table std1 using stdtable tags('beijing')") + tdSql.execute("create table std2 using stdtable tags('shanghai')") + tdSql.execute("create table std3 using stdtable tags('河南')") + tdSql.execute("insert into std1 values(now + 1s, 1)") + tdSql.execute("insert into std1 values(now + 2s, 2);") + tdSql.execute("insert into std2 values(now + 3s, 1);") + tdSql.execute("insert into std2 values(now + 4s, 2);") + tdSql.execute("insert into std3 values(now + 5s, 4);") + tdSql.execute("insert into std3 values(now + 6s, 8);") + tdSql.query("select stddev(col1) from stdtable group by loc;") + tdSql.checkData(0, 0, 2.0) + tdSql.checkData(1, 0, 0.5) + tdSql.checkData(2, 0, 0.5) + + tdSql.execute("create table stdtableint(ts timestamp, col1 int) tags(num int)") + tdSql.execute("create table stdint1 using stdtableint tags(1)") + tdSql.execute("create table stdint2 using stdtableint tags(2)") + tdSql.execute("insert into stdint1 values(now + 1s, 1)") + tdSql.execute("insert into stdint1 values(now + 2s, 2);") + tdSql.execute("insert into stdint2 values(now + 3s, 1);") + tdSql.execute("insert into stdint2 values(now + 4s, 2);") + tdSql.query("select stddev(col1) from stdtableint group by num") + tdSql.checkData(0, 0, 0.5) + tdSql.checkData(1, 0, 0.5) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/functions/queryTestCases.py b/tests/pytest/functions/queryTestCases.py index e8023546c446ff76b5c1976de83c01d4db8f6751..8bdbee09e7f715384f048028e7cccd0010e5642b 100644 --- a/tests/pytest/functions/queryTestCases.py +++ b/tests/pytest/functions/queryTestCases.py @@ -16,6 +16,7 @@ import subprocess import random import math import numpy as np +import inspect from util.log import * from util.cases import * @@ -71,7 +72,6 @@ class TDTestCase: def td4082(self): tdLog.printNoPrefix("==========TD-4082==========") - tdSql.prepare() cfgfile = self.getCfgFile() @@ -84,7 +84,7 @@ class TDTestCase: index_value = np.dstack((cpms_index[0])).squeeze() tdSql.query("show variables") - tdSql.checkData(index_value, 1, -1) + tdSql.checkData(index_value, 1, 524288) tdSql.query("show dnodes") index = tdSql.getData(0, 0) @@ -122,12 +122,8 @@ class TDTestCase: def td4097(self): tdLog.printNoPrefix("==========TD-4097==========") - tdSql.execute("drop database if exists db") tdSql.execute("drop database if exists db1") - tdDnodes.stop(1) - tdDnodes.start(1) - tdSql.execute("create database if not exists db keep 3650") tdSql.execute("create database if not exists db1 keep 3650") tdSql.execute("create database if not exists new keep 3650") @@ -143,7 +139,7 @@ class TDTestCase: tdSql.execute("create table db.t20 using db.stb2 tags(3)") tdSql.execute("create table db1.t30 using db1.stb3 tags(4)") - tdLog.printNoPrefix("==========TD-4097==========") + # tdLog.printNoPrefix("==========TD-4097==========") # 插入数据，然后进行show create 操作 # p1 不进入指定数据库 @@ -279,6 +275,12 @@ class TDTestCase: tdSql.query("show create stable db.stb1") tdSql.checkRows(1) + tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists new") + tdSql.execute("drop database if exists db2") + tdSql.execute("drop database if exists private") + def td4153(self): tdLog.printNoPrefix("==========TD-4153==========") @@ -329,7 +331,6 @@ class TDTestCase: tdSql.checkData(0, 7, 36500) tdSql.execute("drop database if exists db") - tdSql.execute("create database if not exists db1") tdSql.query("show databases") if ("community" in selfPath): @@ -397,29 +398,52 @@ class TDTestCase: def td4889(self): tdLog.printNoPrefix("==========TD-4889==========") + cfg = { + 'minRowsPerFileBlock': '10', + 'maxRowsPerFileBlock': '200', + 'minRows': '10', + 'maxRows': '200', + 'maxVgroupsPerDb': '100', + 'maxTablesPerVnode': '1200', + } + tdSql.query("show dnodes") + dnode_index = tdSql.getData(0,0) + tdDnodes.stop(dnode_index) + tdDnodes.deploy(dnode_index, cfg) + tdDnodes.start(dnode_index) + tdSql.execute("drop database if exists db") - tdSql.execute("create database if not exists db keep 3650") + tdSql.execute("create database if not exists db keep 3650 blocks 3 minrows 10 maxrows 200") tdSql.execute("use db") tdSql.execute("create stable db.stb1 (ts timestamp, c1 int) tags(t1 int)") + nowtime = int(round(time.time() * 1000)) for i in range(1000): tdSql.execute(f"create table db.t1{i} using db.stb1 tags({i})") - for j in range(100): - tdSql.execute(f"insert into db.t1{i} values (now-100d, {i+j})") + sql = f"insert into db.t1{i} values" + for j in range(260): + sql += f"({nowtime-1000*i-j}, {i+j})" + # tdSql.execute(f"insert into db.t1{i} values (now-100d, {i+j})") + tdSql.execute(sql) + + # tdDnodes.stop(dnode_index) + # tdDnodes.start(dnode_index) tdSql.query("show vgroups") index = tdSql.getData(0,0) tdSql.checkData(0, 6, 0) tdSql.execute(f"compact vnodes in({index})") - for i in range(3): + start_time = time.time() + while True: tdSql.query("show vgroups") - if tdSql.getData(0, 6) == 1: + if tdSql.getData(0, 6) != 0: tdLog.printNoPrefix("show vgroups row:0 col:6 data:1 == expect:1") break - if i == 3: + run_time = time.time()-start_time + if run_time > 3: tdLog.exit("compacting not occured") - time.sleep(0.5) + # time.sleep(0.1) pass @@ -445,7 +469,7 @@ class TDTestCase: # tdSql.execute("insert into db.t1 values ('2021-07-01 08:00:04.000', 1001.51, 1001.52, 1001.53, 1001.54)") # for i in range(1000000): - for i in range(1000000): + for i in range(10000): random1 = random.uniform(1000,1001) random2 = random.uniform(1000,1001) random3 = random.uniform(1000,1001) @@ -594,7 +618,7 @@ class TDTestCase: tdSql.execute("use db") tdSql.execute("create stable db.stb1 (ts timestamp, c1 int) tags(t0 tinyint, t1 int)") tdSql.execute("create stable db.stb2 (ts timestamp, c1 int) tags(t0 binary(16), t1 binary(16))") - numtab=2000000 + numtab=20000 for i in range(numtab): sql = f"create table db.t{i} using db.stb1 tags({i%128}, {100+i})" tdSql.execute(sql) @@ -769,7 +793,7 @@ class TDTestCase: tdSql.query(f"select distinct c1,c2 from (select * from t1 where c1 < {tbnum}) ") tdSql.checkRows(3) tdSql.query(f"select distinct c1,c2 from (select * from stb1 where t2 !=0 and t2 != 1) ") - tdSql.checkRows(4) + tdSql.checkRows(0) tdSql.error("select distinct c1, c2 from (select distinct c1, c2 from stb1 where t0 > 2 and t1 < 3) ") tdSql.error("select c1, c2 from (select distinct c1, c2 from stb1 where t0 > 2 and t1 < 3) ") tdSql.query("select distinct c1, c2 from (select c2, c1 from stb1 where c1 > 2 ) where c1 < 4") @@ -1118,27 +1142,448 @@ class TDTestCase: tdSql.error("select ts as t, top(t1, 1) from stb1") tdSql.error("select ts as t, top(t1, 3) from stb1 order by c3") tdSql.error("select ts as t, top(t1, 3) from t1 order by c3") + pass + + def apercentile_query_form(self, col="c1", p=0, com=',', algo="'t-digest'", alias="", table_expr="t1", condition=""): + + ''' + apercentile function: + :param col: string, column name, required parameters; + :param p: float, percentile interval, [0,100], required parameters; + :param algo: string, alforithm, real form like: ', algorithm' , algorithm: {type:int, data:[0, 1]}; + :param alias: string, result column another name; + :param table_expr: string or expression, data source（eg,table/stable name, result set）, required parameters; + :param condition: expression； + :param args: other funtions,like: ', last(col)' + :return: apercentile query statement,default: select apercentile(c1, 0, 1) from t1 + ''' + + return f"select apercentile({col}, {p}{com} {algo}) {alias} from {table_expr} {condition}" + + def checkapert(self,col="c1", p=0, com=',', algo='"t-digest"', alias="", table_expr="t1", condition="" ): + + tdSql.query(f"select count({col}) from {table_expr} {condition}") + if tdSql.queryRows == 0: + tdSql.query(self.apercentile_query_form( + col=col, p=p, com=com, algo=algo, alias=alias, table_expr=table_expr, condition=condition + )) + tdSql.checkRows(0) + return + + pset = [0, 40, 60, 100] + if p not in pset: + pset.append(p) + + if "stb" in table_expr: + tdSql.query(f"select spread({col}) from stb1") + else: + tdSql.query(f"select avg(c1) from (select spread({col.split('.')[-1]}) c1 from stb1 group by tbname)") + spread_num = tdSql.getData(0, 0) + + for pi in pset: + + if "group" in condition: + tdSql.query(f"select last_row({col}) from {table_expr} {condition}") + query_result = tdSql.queryResult + query_rows = tdSql.queryRows + for i in range(query_rows): + pre_condition = condition.replace("slimit",'limit').replace("group by tbname", "").split("soffset")[0] + tbname = query_result[i][-1] + tdSql.query(f"select percentile({col}, {pi}) {alias} from {tbname} {pre_condition}") + print(tdSql.sql) + pre_data = tdSql.getData(0, 0) + tdSql.query(self.apercentile_query_form( + col=col, p=pi, com=com, algo='"t-digest"', alias=alias, table_expr=table_expr, condition=condition + )) + if abs(tdSql.getData(i, 0)) >= (spread_num*0.02): + tdSql.checkDeviaRation(i, 0, pre_data, 0.1) + else: + devia = abs((tdSql.getData(i, 0) - pre_data) / (spread_num * 0.02)) + if devia < 0.5: + tdLog.info(f"sql:{tdSql.sql}, result data:{tdSql.getData(i, 0)}, expect data:{pre_data}, " + f"actual deviation:{devia} <= expect deviation: 0.01") + else: + tdLog.exit( + f"[{inspect.getframeinfo(inspect.stack()[1][0]).lineno}],check failed:sql:{tdSql.sql}, " + f"result data:{tdSql.getData(i, 0)}, expect data:{pre_data}, " + f"actual deviation:{devia} > expect deviation: 0.01") + + # if "group" in condition: + # tdSql.query(self.apercentile_query_form( + # col=col, p=pi, com=com, algo='"default"', alias=alias, table_expr=table_expr, condition=condition + # )) + # query_result = tdSql.queryResult + # query_rows = tdSql.queryRows + # tdSql.query(self.apercentile_query_form( + # col=col, p=pi, com=com, algo='"t-digest"', alias=alias, table_expr=table_expr, condition=condition + # )) + # for i in range(query_rows): + # if abs(tdSql.getData(i, 0)) >= (spread_num*0.02): + # tdSql.checkDeviaRation(i, 0, query_result[i][0], 0.1) + # else: + # devia = abs((tdSql.getData(i, 0) - query_result[i][0]) / (spread_num * 0.02)) + # if devia < 0.5: + # tdLog.info(f"sql:{tdSql.sql}, result data:{tdSql.getData(i, 0)}, expect data:{tdSql.queryResult[i][0]}, " + # f"actual deviation:{devia} <= expect deviation: 0.01") + # else: + # tdLog.exit( + # f"[{inspect.getframeinfo(inspect.stack()[1][0]).lineno}],check failed:sql:{tdSql.sql}, " + # f"result data:{tdSql.getData(i, 0)}, expect data:{tdSql.queryResult[i][0]}, " + # f"actual deviation:{devia} > expect deviation: 0.01") + + else: + if ',' in alias or not alias: + tdSql.query(f"select {col} from {table_expr} {condition}") + elif "stb" not in table_expr: + tdSql.query(f"select percentile({col}, {pi}) {alias} from {table_expr} {condition}") + else: + tdSql.query(self.apercentile_query_form( + col=col, p=pi, com=com, algo='"default"', alias=alias, table_expr=table_expr, condition=condition + )) + query_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + tdSql.query(self.apercentile_query_form( + col=col, p=pi, com=com, algo=algo, alias=alias, table_expr=table_expr, condition=condition + )) + + if abs(tdSql.getData(0, 0)) >= (spread_num * 0.02): + tdSql.checkDeviaRation(0, 0, np.percentile(query_result, pi), 0.1) + else: + devia = abs((tdSql.getData(0, 0) - np.percentile(query_result, pi)) / (spread_num * 0.02)) + if devia < 0.5: + tdLog.info( + f"sql:{tdSql.sql}, result data:{tdSql.getData(0, 0)}, expect data:{np.percentile(query_result, pi)}, " + f"actual deviation:{devia} <= expect deviation: 0.01") + else: + tdLog.exit( + f"[{inspect.getframeinfo(inspect.stack()[1][0]).lineno}],check failed:sql:{tdSql.sql}, " + f"result data:{tdSql.getData(0, 0)}, expect data:{np.percentile(query_result, pi)}, " + f"actual deviation:{devia} > expect deviation: 0.01") + + + def apercentile_query(self): + + # table schema :ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool + # c7 bigint, c8 smallint, c9 tinyint, c10 nchar(16) + + # case1： int col + self.checkapert() + # case2: float col + case2 = {'col':'c2'} + self.checkapert(**case2) + # case3: double col + case3 = {'col':'c5'} + self.checkapert(**case3) + # case4: bigint col + case4 = {'col':'c7'} + self.checkapert(**case4) + # case5: smallint col + case5 = {'col':'c8'} + self.checkapert(**case5) + # case6: tinyint col + case6 = {'col':'c9'} + self.checkapert(**case6) + # case7: stable + case7 = {'table_expr':'stb1'} + self.checkapert(**case7) + # case8: nest query, outquery + case8 = {'table_expr':'(select c1 from t1)'} + self.checkapert(**case8) + # case9: nest query, inquery and out query + case9 = {'table_expr':'(select apercentile(c1, 0) as c1 from t1)'} + self.checkapert(**case9) + + # case10: nest query, inquery + tdSql.query("select * from (select c1 from stb1)") + if tdSql.queryRows == 0: + tdSql.query("select * from (select apercentile(c1,0) c1 from stb1)") + tdSql.checkRows(0) + else: + query_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + tdSql.query("select * from (select apercentile(c1, 0) c1 from stb1)") + tdSql.checkDeviaRation(0, 0, np.percentile(query_result, 0), 0.1) + tdSql.query("select * from (select apercentile(c1,100) c1 from stb1)") + tdSql.checkDeviaRation(0, 0, np.percentile(query_result, 100), 0.1) + tdSql.query("select * from (select apercentile(c1,40) c1 from stb1)") + tdSql.checkDeviaRation(0, 0, np.percentile(query_result, 40), 0.1) + + # case11: no algorithm = algo:0 + case11 = {'com':'', 'algo': ''} + self.checkapert(**case11) + + # case12~14: p: bin/oct/hex + case12 = {'p': 0b1100100} + self.checkapert(**case12) + case13 = {'algo':'"T-DIGEST"'} + self.checkapert(**case13) + case14 = {'p':0x32, 'algo':'"DEFAULT"'} + self.checkapert(**case14) + + # case15~21: mix with aggregate function + case15 = {'alias':', count(*)'} + self.checkapert(**case15) + case16 = {'alias':', avg(c1)'} + self.checkapert(**case16) + case17 = {'alias':', twa(c1)'} + self.checkapert(**case17) + case18 = {'alias':', irate(c1)'} + self.checkapert(**case18) + case19 = {'alias':', sum(c1)'} + self.checkapert(**case19) + case20 = {'alias':', stddev(c1)'} + self.checkapert(**case20) + case21 = {'alias':', leastsquares(c1, 1, 1)'} + self.checkapert(**case21) + + # case22~27：mix with selector function + case22 = {'alias':', min(c1)'} + self.checkapert(**case22) + case23 = {'alias':', max(c1)'} + self.checkapert(**case23) + case24 = {'alias':', first(c1)'} + self.checkapert(**case24) + case25 = {'alias':', last(c1)'} + self.checkapert(**case25) + case26 = {'alias':', percentile(c1, 0)'} + self.checkapert(**case26) + case27 = {'alias':', apercentile(c1, 0, "t-digest")'} + self.checkapert(**case27) + + # case28~29: mix with computing function + case28 = {'alias':', spread(c1)'} + self.checkapert(**case28) + # case29: mix with four operation + case29 = {'alias':'+ spread(c1)'} + self.checkapert(**case29) + + # case30~36: with condition + case30 = {'condition':'where ts > now'} + self.checkapert(**case30) + case31 = {'condition':'where c1 between 1 and 200'} + self.checkapert(**case31) + case32 = {'condition':f'where c1 in {tuple(i for i in range(200))}'} + self.checkapert(**case32) + case33 = {'condition':'where c1>100 and c2<100'} + self.checkapert(**case33) + case34 = {'condition':'where c1 is not null'} + self.checkapert(**case34) + case35 = {'condition':'where c4 like "_inary%"'} + self.checkapert(**case35) + case36 = {'table_expr':'stb1' ,'condition':'where tbname like "t_"'} + self.checkapert(**case36) + + # case37~38: with join + case37 = {'col':'t1.c1','table_expr':'t1, t2 ','condition':'where t1.ts=t2.ts'} + self.checkapert(**case37) + case38 = {'col':'stb1.c1', 'table_expr':'stb1, stb2', 'condition':'where stb1.ts=stb2.ts and stb1.st1=stb2.st2'} + self.checkapert(**case38) + + # case39: with group by + case39 = {'table_expr':'stb1', 'condition':'group by tbname'} + self.checkapert(**case39) + + # case40: with slimit + case40 = {'table_expr':'stb1', 'condition':'group by tbname slimit 1'} + self.checkapert(**case40) + + # case41: with soffset + case41 = {'table_expr':'stb1', 'condition':'group by tbname slimit 1 soffset 1'} + self.checkapert(**case41) + + # case42: with order by + case42 = {'table_expr':'stb1' ,'condition':'order by ts'} + self.checkapert(**case42) + case43 = {'table_expr':'t1' ,'condition':'order by ts'} + self.checkapert(**case43) + + # case44: with limit offset + case44 = {'table_expr':'stb1', 'condition':'group by tbname limit 1'} + self.checkapert(**case44) + case45 = {'table_expr':'stb1', 'condition':'group by tbname limit 1 offset 1'} + self.checkapert(**case45) pass + def error_apercentile(self): + + # unusual test + # + # table schema :ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool + # c7 bigint, c8 smallint, c9 tinyint, c10 nchar(16) + # + # form test + tdSql.error(self.apercentile_query_form(col="",com='',algo='')) # no col , no algorithm + tdSql.error(self.apercentile_query_form(col="")) # no col , algorithm + tdSql.error(self.apercentile_query_form(p='',com='',algo='')) # no p , no algorithm + tdSql.error(self.apercentile_query_form(p='')) # no p , algorithm + tdSql.error("apercentile( c1, 100) from t1") # no select + tdSql.error("select apercentile from t1") # no algorithm condition + tdSql.error("select apercentile c1,0 from t1") # no brackets + tdSql.error("select apercentile (c1,0) t1") # no from + tdSql.error(self.apercentile_query_form(col='(c1,0)',p='',com='',algo='')) # no p , no algorithm + tdSql.error("select apercentile( (c1,0) ) from t1") # no table_expr + tdSql.error("select apercentile{ (c1,0) } from t1") # sql form error 1 + tdSql.error("select apercentile[ (c1,0) ] from t1") # sql form error 2 + tdSql.error("select [apercentile(c1,0) ] from t1") # sql form error 3 + tdSql.error("select apercentile((c1, 0), 'default') from t1") # sql form error 5 + tdSql.error("select apercentile(c1, (0, 'default')) from t1") # sql form error 6 + tdSql.error("select apercentile(c1, (0), 1) from t1") # sql form error 7 + tdSql.error("select apercentile([c1, 0], 'default') from t1") # sql form error 8 + tdSql.error("select apercentile(c1, [0, 'default']) from t1") # sql form error 9 + tdSql.error("select apercentile(c1, {0, 'default'}) from t1") # sql form error 10 + tdSql.error("select apercentile([c1, 0]) from t1") # sql form error 11 + tdSql.error("select apercentile({c1, 0}) from t1") # sql form error 12 + tdSql.error("select apercentile(c1) from t1") # agrs: 1 + tdSql.error("select apercentile(c1, 0, 'default', 0) from t1") # agrs: 4 + tdSql.error("select apercentile(c1, 0, 0, 'default') from t1") # agrs: 4 + tdSql.error("select apercentile() from t1") # agrs: null 1 + tdSql.error("select apercentile from t1") # agrs: null 2 + tdSql.error("select apercentile( , , ) from t1") # agrs: null 3 + tdSql.error(self.apercentile_query_form(col='', p='', algo='')) # agrs: null 4 + tdSql.error(self.apercentile_query_form(col="st1")) # col:tag column + tdSql.error(self.apercentile_query_form(col=123)) # col:numerical + tdSql.error(self.apercentile_query_form(col=True)) # col:bool + tdSql.error(self.apercentile_query_form(col='')) # col:'' + tdSql.error(self.apercentile_query_form(col="last(c1)")) # col:expr + tdSql.error(self.apercentile_query_form(col="t%")) # col:non-numerical + tdSql.error(self.apercentile_query_form(col="c3")) # col-type: timestamp + tdSql.error(self.apercentile_query_form(col="c4")) # col-type: binary + tdSql.error(self.apercentile_query_form(col="c6")) # col-type: bool + tdSql.error(self.apercentile_query_form(col="c10")) # col-type: nchar + tdSql.error(self.apercentile_query_form(p=True)) # p:bool + tdSql.error(self.apercentile_query_form(p='a')) # p:str + tdSql.error(self.apercentile_query_form(p='last(*)')) # p:expr + tdSql.error(self.apercentile_query_form(p="2021-08-01 00:00:00.000")) # p:timestamp + tdSql.error(self.apercentile_query_form(algo='t-digest')) # algorithm:str + tdSql.error(self.apercentile_query_form(algo='"t_digest"')) # algorithm:str + tdSql.error(self.apercentile_query_form(algo='"t-digest0"')) # algorithm:str + tdSql.error(self.apercentile_query_form(algo='"t-digest."')) # algorithm:str + tdSql.error(self.apercentile_query_form(algo='"t-digest%"')) # algorithm:str + tdSql.error(self.apercentile_query_form(algo='"t-digest*"')) # algorithm:str + tdSql.error(self.apercentile_query_form(algo='tdigest')) # algorithm:str + tdSql.error(self.apercentile_query_form(algo=2.0)) # algorithm:float + tdSql.error(self.apercentile_query_form(algo=1.9999)) # algorithm:float + tdSql.error(self.apercentile_query_form(algo=-0.9999)) # algorithm:float + tdSql.error(self.apercentile_query_form(algo=-1.0)) # algorithm:float + tdSql.error(self.apercentile_query_form(algo=0b1)) # algorithm:float + tdSql.error(self.apercentile_query_form(algo=0x1)) # algorithm:float + tdSql.error(self.apercentile_query_form(algo=0o1)) # algorithm:float + tdSql.error(self.apercentile_query_form(algo=True)) # algorithm:bool + tdSql.error(self.apercentile_query_form(algo="True")) # algorithm:bool + tdSql.error(self.apercentile_query_form(algo='2021-08-01 00:00:00.000')) # algorithm:timestamp + tdSql.error(self.apercentile_query_form(algo='last(c1)')) # algorithm:expr + + # boundary test + tdSql.error(self.apercentile_query_form(p=-1)) # p left out of [0, 100] + tdSql.error(self.apercentile_query_form(p=-9223372036854775809)) # p left out of bigint + tdSql.error(self.apercentile_query_form(p=100.1)) # p right out of [0, 100] + tdSql.error(self.apercentile_query_form(p=18446744073709551616)) # p right out of unsigned-bigint + tdSql.error(self.apercentile_query_form(algo=-1)) # algorithm left out of [0, 1] + tdSql.error(self.apercentile_query_form(algo=-9223372036854775809)) # algorithm left out of unsigned-bigint + tdSql.error(self.apercentile_query_form(algo=2)) # algorithm right out of [0, 1] + tdSql.error(self.apercentile_query_form(algo=18446744073709551616)) # algorithm right out of unsigned-bigint + + # mix function test + tdSql.error(self.apercentile_query_form(alias=', top(c1,1)')) # mix with top function + tdSql.error(self.apercentile_query_form(alias=', top(c1,1)')) # mix with bottom function + tdSql.error(self.apercentile_query_form(alias=', last_row(c1)')) # mix with last_row function + tdSql.error(self.apercentile_query_form(alias=', distinct c1 ')) # mix with distinct function + tdSql.error(self.apercentile_query_form(alias=', *')) # mix with * + tdSql.error(self.apercentile_query_form(alias=', diff(c1)')) # mix with diff function + tdSql.error(self.apercentile_query_form(alias=', interp(c1)', condition='ts="2021-10-10 00:00:00.000"')) # mix with interp function + tdSql.error(self.apercentile_query_form(alias=', derivative(c1, 10m, 0)')) # mix with derivative function + tdSql.error(self.apercentile_query_form(alias=', diff(c1)')) # mix with diff function + tdSql.error(self.apercentile_query_form(alias='+ c1)')) # mix with four operation + + def apercentile_data(self, tbnum, data_row, basetime): + for i in range(tbnum): + for j in range(data_row): + tdSql.execute( + f"insert into t{i} values (" + f"{basetime + j*10}, {random.randint(-200, -1)}, {random.uniform(200, -1)}, {basetime + random.randint(-200, -1)}, " + f"'binary_{j}', {random.uniform(-200, -1)}, {random.choice([0,1])}, {random.randint(-200,-1)}, " + f"{random.randint(-200, -1)}, {random.randint(-127, -1)}, 'nchar_{j}' )" + ) + + tdSql.execute( + f"insert into t{i} values (" + f"{basetime - (j+1) * 10}, {random.randint(1, 200)}, {random.uniform(1, 200)}, {basetime - random.randint(1, 200)}, " + f"'binary_{j}_1', {random.uniform(1, 200)}, {random.choice([0, 1])}, {random.randint(1,200)}, " + f"{random.randint(1,200)}, {random.randint(1,127)}, 'nchar_{j}_1' )" + ) + tdSql.execute( + f"insert into tt{i} values ( {basetime-(j+1) * 10}, {random.randint(1, 200)} )" + ) + + pass + + def td6108(self): + tdLog.printNoPrefix("==========TD-6108==========") + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 3650") + tdSql.execute("use db") + + tdSql.execute( + "create stable db.stb1 (\ + ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool, \ + c7 bigint, c8 smallint, c9 tinyint, c10 nchar(16)\ + ) \ + tags(st1 int)" + ) + tdSql.execute( + "create stable db.stb2 (ts timestamp, c1 int) tags(st2 int)" + ) + tbnum = 10 + for i in range(tbnum): + tdSql.execute(f"create table t{i} using stb1 tags({i})") + tdSql.execute(f"create table tt{i} using stb2 tags({i})") + + tdLog.printNoPrefix("######## no data test:") + self.apercentile_query() + self.error_apercentile() + + tdLog.printNoPrefix("######## insert data test:") + nowtime = int(round(time.time() * 1000)) + per_table_rows = 1000 + self.apercentile_data(tbnum, per_table_rows, nowtime) + self.apercentile_query() + self.error_apercentile() + + tdLog.printNoPrefix("######## insert data with NULL test:") + tdSql.execute(f"insert into t1(ts) values ({nowtime-5})") + tdSql.execute(f"insert into t1(ts) values ({nowtime+5})") + self.apercentile_query() + self.error_apercentile() + + tdLog.printNoPrefix("######## check after WAL test:") + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + + self.apercentile_query() + self.error_apercentile() + def run(self): + self.td4097() + # master branch self.td3690() - self.td4082() + # self.td4082() self.td4288() self.td4724() self.td5935() self.td6068() - # develop branch - # self.td4097() - # self.td4889() # self.td5168() # self.td5433() # self.td5798() + # develop branch + # self.td4889() In the scenario that with vnode/wal/wal* but without meta/data in vnode, the status is reset to 0 right now. + self.td5798() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/pytest/functions/variable_httpDbNameMandatory.py b/tests/pytest/functions/variable_httpDbNameMandatory.py new file mode 100644 index 0000000000000000000000000000000000000000..1cd2516ec9303f2f038bc22a69afcd01bce3a930 --- /dev/null +++ b/tests/pytest/functions/variable_httpDbNameMandatory.py @@ -0,0 +1,152 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import subprocess +import random +import math +import numpy as np +import inspect + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +from requests.auth import HTTPBasicAuth +import requests +import json + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self) -> str: + selfPath = os.path.dirname(os.path.realpath(__file__)) + global cfgPath + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + cfgPath = projPath + "/community/sim/dnode1/cfg" + else: + projPath = selfPath[:selfPath.find("tests")] + cfgPath = projPath + "/sim/dnode1/cfg" + + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def getCfgDir(self) -> str: + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + cfgDir = self.getBuildPath() + "/community/sim/dnode1/cfg" + + else: + cfgDir = self.getBuildPath() + "/sim/dnode1/cfg" + + return cfgDir + + def getCfgFile(self) -> str: + return self.getCfgDir()+"/taos.cfg" + + def rest_query(self,sql,db=''): + host = '127.0.0.1' + user = 'root' + password = 'taosdata' + port =6041 + if db == '': + url = "http://{}:{}/rest/sql".format(host, port ) + else: + url = "http://{}:{}/rest/sql/{}".format(host, port, db ) + try: + r = requests.post(url, + data = 'use db' , + auth = HTTPBasicAuth('root', 'taosdata')) + r = requests.post(url, + data = sql, + auth = HTTPBasicAuth('root', 'taosdata')) + except: + print("REST API Failure (TODO: more info here)") + raise + rj = dict(r.json()['data']) + return rj + + def TS834(self): + tdLog.printNoPrefix("==========TS-782==========") + tdSql.prepare() + buildPath = self.getBuildPath() + cfgfile = cfgPath + "/taos.cfg" + + tdSql.execute("show variables") + res_com = tdSql.cursor.fetchall() + rescomlist = np.array(res_com) + cpms_index = np.where(rescomlist == "httpDbNameMandatory") + index_value = np.dstack((cpms_index[0])).squeeze() + + tdSql.query("show variables") + tdSql.checkData(index_value, 1, 0) + + rj = self.rest_query("show variables") + if 'httpDbNameMandatory' not in rj: + tdLog.info('has no httpDbNameMandatory shown') + tdLog.exit(1) + if rj['httpDbNameMandatory'] != '0': + tdLog.info('httpDbNameMandatory data:%s == expect:0'%rj['httpDbNameMandatory']) + tdLog.exit(1) + tdLog.info("httpDbNameMandatory by restful query data:%s == expect:0" % (rj['httpDbNameMandatory'])) + + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdLog.info("restart taosd ") + tdDnodes.stop(index) + cmd = f"echo 'httpDbNameMandatory 1' >> {cfgfile} " + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except Exception as e: + raise e + + tdDnodes.start(index) + tdSql.query("show variables") + tdSql.checkData(index_value, 1, 1) + + rj = self.rest_query("show variables", 'db') + if 'httpDbNameMandatory' not in rj: + tdLog.info('has no httpDbNameMandatory shown') + tdLog.exit(1) + if rj['httpDbNameMandatory'] != '1': + tdLog.info('httpDbNameMandatory data:%s == expect:0'%rj['httpDbNameMandatory']) + tdLog.exit(1) + tdLog.info("httpDbNameMandatory by restful query data:%s == expect:1" % (rj['httpDbNameMandatory'])) + + def run(self): + + #TS-834 https://jira.taosdata.com:18080/browse/TS-834 + self.TS834() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + + + diff --git a/tests/pytest/import_merge/import_update_0.py b/tests/pytest/import_merge/import_update_0.py index a81fd0e497e2732789670e839b1206165c484bfb..f24a4739644fffe14fd9273ee4501d91a3447cfb 100644 --- a/tests/pytest/import_merge/import_update_0.py +++ b/tests/pytest/import_merge/import_update_0.py @@ -414,7 +414,7 @@ class TDTestCase: tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary%s', 'nchar%s', %f, %f, %d)''' % (self.ts + i, random.randint(-2147483647, 2147483647), random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.randint(-127, 127), random.randint(-99, 99), random.randint(-9999, 9999), random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' @@ -422,7 +422,7 @@ class TDTestCase: tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, false, 'binary%s', 'nchar%s', %f, %f, %d)''' % (self.ts + i, random.randint(-2147483647, 2147483647), random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.randint(-127, 127), random.randint(-99, 99), random.randint(-9999, 9999), random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) tdLog.info("========== regular_table ==========") @@ -1910,4 +1910,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/import_update_1.py b/tests/pytest/import_merge/import_update_1.py index 8a17b1d8a311d5af4a9f58a57e5395bd622a93c1..b3f4b46e11549f934069e32349c0f72050ae21f8 100644 --- a/tests/pytest/import_merge/import_update_1.py +++ b/tests/pytest/import_merge/import_update_1.py @@ -414,7 +414,7 @@ class TDTestCase: tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary%s', 'nchar%s', %f, %f, %d)''' % (self.ts + i, random.randint(-2147483647, 2147483647), random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.randint(-127, 127), random.randint(-99, 99), random.randint(-9999, 9999), random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' @@ -422,7 +422,7 @@ class TDTestCase: tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, false, 'binary%s', 'nchar%s', %f, %f, %d)''' % (self.ts + i, random.randint(-2147483647, 2147483647), random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.randint(-127, 127), random.randint(-99, 99), random.randint(-9999, 9999), random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) tdLog.info("========== regular_table ==========") @@ -1910,4 +1910,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/import_update_2.py b/tests/pytest/import_merge/import_update_2.py index 9a7b9e6b0d30ce7254a71d8b5d3a0563a3851dff..16328956c3dcea62125cd9fe3817aa68d79362f3 100644 --- a/tests/pytest/import_merge/import_update_2.py +++ b/tests/pytest/import_merge/import_update_2.py @@ -414,7 +414,7 @@ class TDTestCase: tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, false, 'binary%s', 'nchar%s', %f, %f, %d)''' % (self.ts + i, random.randint(-2147483647, 2147483647), random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.randint(-127, 127), random.randint(-99, 99), random.randint(-9999, 9999), random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' @@ -422,7 +422,7 @@ class TDTestCase: tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, false, 'binary%s', 'nchar%s', %f, %f, %d)''' % (self.ts + i, random.randint(-2147483647, 2147483647), random.randint(-9223372036854775807, 9223372036854775807), random.randint(-32767, 32767), - random.randint(-127, 127), random.randint(-100, 100), random.randint(-10000, 10000), + random.randint(-127, 127), random.randint(-99, 99), random.randint(-9999, 9999), random.uniform(-100000,100000), random.uniform(-1000000000,1000000000), self.ts + i)) tdLog.info("========== regular_table ==========") @@ -2270,4 +2270,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/timestamp.py b/tests/pytest/insert/timestamp.py new file mode 100644 index 0000000000000000000000000000000000000000..4c9cf36f40e31a792b550a557937d25d968f4fe2 --- /dev/null +++ b/tests/pytest/insert/timestamp.py @@ -0,0 +1,81 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.ts = 1607281690000 + + def run(self): + tdSql.prepare() + + # TS-806 + tdLog.info("test case for TS-806") + + # Case 1 + tdSql.execute("create table t1(ts timestamp, c1 int)") + tdSql.execute("insert into t1(c1, ts) values(1, %d)" % self.ts) + tdSql.query("select * from t1") + tdSql.checkRows(1) + + # Case 2 + tdSql.execute("insert into t1(c1, ts) values(2, %d)(3, %d)" % (self.ts + 1000, self.ts + 2000)) + tdSql.query("select * from t1") + tdSql.checkRows(3) + + # Case 3 + tdSql.execute("create table t2(ts timestamp, c1 timestamp)") + tdSql.execute(" insert into t2(c1, ts) values(%d, %d)" % (self.ts, self.ts + 5000)) + tdSql.query("select * from t2") + tdSql.checkRows(1) + + tdSql.execute(" insert into t2(c1, ts) values(%d, %d)(%d, %d)" % (self.ts, self.ts + 6000, self.ts + 3000, self.ts + 8000)) + tdSql.query("select * from t2") + tdSql.checkRows(3) + + # Case 4 + tdSql.execute("create table stb(ts timestamp, c1 int, c2 binary(20)) tags(tstag timestamp, t1 int)") + tdSql.execute("insert into tb1(c2, ts, c1) using stb(t1, tstag) tags(1, now) values('test', %d, 1)" % self.ts) + tdSql.query("select * from stb") + tdSql.checkRows(1) + + # Case 5 + tdSql.execute("insert into tb1(c2, ts, c1) using stb(t1, tstag) tags(1, now) values('test', now, 1) tb2(c1, ts) using stb tags(now + 2m, 1000) values(1, now - 1h)") + tdSql.query("select * from stb") + tdSql.checkRows(3) + + tdSql.execute(" insert into tb1(c2, ts, c1) using stb(t1, tstag) tags(1, now) values('test', now + 10s, 1) tb2(c1, ts) using stb(tstag) tags(now + 2m) values(1, now - 3h)(2, now - 2h)") + tdSql.query("select * from stb") + tdSql.checkRows(6) + + # Case 6 + tdSql.execute("create table stb2 (ts timestamp, c1 timestamp, c2 timestamp) tags(t1 timestamp, t2 timestamp)") + tdSql.execute(" insert into tb4(c1, c2, ts) using stb2(t2, t1) tags(now, now + 1h) values(now + 1s, now + 2s, now + 3s)(now -1s, now - 2s, now - 3s) tb5(c2, ts, c1) using stb2(t2) tags(now + 1h) values(now, now, now)") + tdSql.query("select * from stb2") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/bug5903.py b/tests/pytest/query/bug5903.py new file mode 100644 index 0000000000000000000000000000000000000000..a973efbbdf7a0868a4db7910bf57bd5eef8dae06 --- /dev/null +++ b/tests/pytest/query/bug5903.py @@ -0,0 +1,36 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + # TD-5903 show db.vgroups xxx. xxx is invalid content, but still returns results. + + tdSql.execute("create database if not exists test_5903") + tdSql.execute("show test_5903.vgroups") + tdSql.error("show test_5903.vgroups xxx") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/nestedQuery/nestedQuery.py b/tests/pytest/query/nestedQuery/nestedQuery.py index 545f6429e825c468bdb07524329d6ea49944e379..2b6f8a63550ecb35907dca71cf94437cc2364605 100755 --- a/tests/pytest/query/nestedQuery/nestedQuery.py +++ b/tests/pytest/query/nestedQuery/nestedQuery.py @@ -470,9 +470,9 @@ class TDTestCase: #inter && calc_aggregate_all\calc_aggregate_regular\calc_select_all interval_sliding = ['interval(4w) sliding(1w) ','interval(1w) sliding(1d) ','interval(1d) sliding(1h) ' , - 'interval(1h) sliding(1m) ','interval(1m) sliding(1s) ','interval(1s) sliding(10a) ', - 'interval(1y) ','interval(1n) ','interval(1w) ','interval(1d) ','interval(1h) ','interval(1m) ','interval(1s) ' ,'interval(10a)', - 'interval(1y,1n) ','interval(1n,1w) ','interval(1w,1d) ','interval(1d,1h) ','interval(1h,1m) ','interval(1m,1s) ','interval(1s,10a) ' ,'interval(100a,30a)'] + 'interval(1h) sliding(1m) ','interval(1m) sliding(1s) ','interval(1s) sliding(100a) ', + 'interval(1y) ','interval(1n) ','interval(1w) ','interval(1d) ','interval(1h) ','interval(1m) ','interval(1s) ' ,'interval(100a)', + 'interval(1y,1n) ','interval(1n,1w) ','interval(1w,1d) ','interval(1d,1h) ','interval(1h,1m) ','interval(1m,1s) ','interval(1s,100a) ' ,'interval(100a,30a)'] #1 select * from (select column form regular_table where <\>\in\and\or order by) tdSql.query("select 1-1 from table_0;") @@ -488,11 +488,10 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) #tdSql.checkData(0,0,'2020-09-13 20:26:40.000') - tdSql.checkRows(6*self.num) - + tdSql.checkRows(6*self.num) #1 outer union not support - dcDB = self.dropandcreateDB(random.randint(1,3)) + #dcDB = self.dropandcreateDB(random.randint(1,3)) tdSql.query("select 1-2 from table_0;") for i in range(self.fornum): sql = "select ts , * from ( select " @@ -2114,6 +2113,119 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 21-1 from table_0;") + for i in range(self.fornum): + sql = "select avg(res1),min(res2),max(res3) from ( select " + sql += "%s res1, " % random.choice(calc_aggregate_all) + sql += "%s res2," % random.choice(calc_aggregate_all) + sql += "%s res3 " % random.choice(calc_aggregate_all) + sql += " from regular_table_1 t1 " + sql += " where %s " % random.choice(q_where) + sql += " %s ) " % random.choice(interval_sliding) + sql += " where ts >now-10h and ts < now+10h " + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(fill_where) + sql += "%s ;" % random.choice(limit_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + tdSql.query("select 21-2 from table_0;") + for i in range(self.fornum): + sql = "select avg(res1),min(res2),max(res3) from ( select " + sql += "%s res1, " % random.choice(calc_aggregate_all) + sql += "%s res2," % random.choice(calc_aggregate_all) + sql += "%s res3 " % random.choice(calc_aggregate_all) + sql += " from table_1 t1 " + sql += " where %s " % random.choice(q_where) + sql += " %s ) " % random.choice(interval_sliding) + sql += " where ts >now-10h and ts < now+10h " + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(fill_where) + sql += "%s ;" % random.choice([limit_where[2] , limit_where[3]] ) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + tdSql.query("select 21-3 from table_0;") + for i in range(self.fornum): + sql = "select avg(res1),min(res2),max(res3) from ( select " + sql += "%s res1, " % random.choice(calc_aggregate_all) + sql += "%s res2," % random.choice(calc_aggregate_all) + sql += "%s res3 " % random.choice(calc_aggregate_all) + sql += " from stable_1 t1 " + sql += " where %s " % random.choice(q_where) + sql += " %s ) " % random.choice(interval_sliding) + sql += " where ts >now-10h and ts < now+10h " + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(fill_where) + sql += "%s ;" % random.choice(limit_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + tdSql.query("select 21-4 from table_0;") + for i in range(self.fornum): + sql = "select avg(res1),min(res2),max(res3) from ( select " + sql += "%s res1, " % random.choice(calc_aggregate_all) + sql += "%s res2," % random.choice(calc_aggregate_all) + sql += "%s res3 " % random.choice(calc_aggregate_all) + sql += " from regular_table_1 t1 " + sql += " where %s " % random.choice(q_where) + sql += " %s ) " % random.choice(interval_sliding) + sql += "group by ts " + sql += "%s ;" % random.choice(limit_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + tdSql.query("select 21-5 from table_0;") + for i in range(self.fornum): + sql = "select avg(res1),min(res2),max(res3) from ( select " + sql += "%s res1, " % random.choice(calc_aggregate_all) + sql += "%s res2," % random.choice(calc_aggregate_all) + sql += "%s res3 " % random.choice(calc_aggregate_all) + sql += " from table_1 t1 " + sql += " where %s " % random.choice(q_where) + sql += " %s ) " % random.choice(interval_sliding) + sql += "group by ts " + sql += "%s ;" % random.choice([limit_where[2] , limit_where[3]] ) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + tdSql.query("select 21-6 from table_0;") + for i in range(self.fornum): + sql = "select avg(res1),min(res2),max(res3) from ( select " + sql += "%s res1, " % random.choice(calc_aggregate_all) + sql += "%s res2," % random.choice(calc_aggregate_all) + sql += "%s res3 " % random.choice(calc_aggregate_all) + sql += " from stable_1 t1 " + sql += " where %s " % random.choice(q_where) + sql += " %s ) " % random.choice(interval_sliding) + sql += "group by ts " + sql += "%s ;" % random.choice(limit_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + tdSql.query("select 21-7 from table_0;") + for i in range(self.fornum): + sql = "select avg(res1),min(res2),max(res3) from ( select " + sql += "%s res1, " % random.choice(calc_aggregate_all) + sql += "%s res2," % random.choice(calc_aggregate_all) + sql += "%s res3 " % random.choice(calc_aggregate_all) + sql += " from stable_1 t1 " + sql += " where %s " % random.choice(q_where) + sql += " %s " % random.choice(interval_sliding) + sql += " %s ) " % random.choice(group_where) + sql += "group by ts " + sql += "%s ;" % random.choice(limit_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + # error #1 select * from (select * from (select * form regular_table where <\>\in\and\or order by limit )) tdSql.query("select 1-1 from table_1;") diff --git a/tests/pytest/query/nestedQuery/nestedQuery_datacheck.py b/tests/pytest/query/nestedQuery/nestedQuery_datacheck.py new file mode 100755 index 0000000000000000000000000000000000000000..308bf4f9e69828bf80728e320247a03303c7121e --- /dev/null +++ b/tests/pytest/query/nestedQuery/nestedQuery_datacheck.py @@ -0,0 +1,637 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +import os +import sys +import time +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + updatecfgDict={'maxSQLLength':1048576} + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + os.system("rm -rf query/nestedQuery/nestedQuery_datacheck.py.sql") + now = time.time() + self.ts = 1630000000000 + self.num = 100 + self.fornum = 3 + + def get_random_string(self, length): + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(length)) + return result_str + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def restartDnodes(self): + tdDnodes.stop(1) + tdDnodes.start(1) + + def dropandcreateDB(self,n): + for i in range(n): + tdSql.execute('''drop database if exists db ;''') + tdSql.execute('''create database db keep 36500;''') + tdSql.execute('''use db;''') + + tdSql.execute('''create stable stable_1 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) ,q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute('''create stable stable_2 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) ,q_float float , q_double double , q_ts timestamp) + tags(loc nchar(20) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, + t_bool bool , t_binary binary(20) , t_nchar nchar(20) ,t_float float , t_double double , t_ts timestamp);''') + + + tdSql.execute('''create table table_0 using stable_1 + tags('table_0' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')''') + tdSql.execute('''create table table_1 using stable_1 + tags('table_1' , '2147483647' , '9223372036854775807' , '32767' , '127' , 1 , + 'binary1' , 'nchar1' , '1' , '11' , \'1999-09-09 09:09:09.090\')''') + tdSql.execute('''create table table_2 using stable_1 + tags('table_2' , '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , + 'binary2' , 'nchar2nchar2' , '-2.2' , '-22.22' , \'2099-09-09 09:09:09.090\')''') + tdSql.execute('''create table table_21 using stable_2 + tags('table_21' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')''') + + #regular table + tdSql.execute('''create table regular_table_1 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) ,q_float float , q_double double , q_ts timestamp) ;''') + tdSql.execute('''create table regular_table_2 + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint, + q_bool bool , q_binary binary(20) , q_nchar nchar(20) ,q_float float , q_double double , q_ts timestamp) ;''') + + + for i in range(self.num): + tdSql.execute('''insert into table_0 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i*10000000, i, i, i, i, i, i, i, i, self.ts + i)) + tdSql.execute('''insert into table_1 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' + % (self.ts + i*10000000+1, 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, + i, i, i, i, 1262304000001 + i)) + tdSql.execute('''insert into table_2 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' + % (self.ts + i*10000000+2, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, + i, i, -i, -i, 1577836800001 + i)) + + tdSql.execute('''insert into table_21 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i*10000000, i, i, i, i, i, i, i, i, self.ts + i)) + + tdSql.execute('''insert into regular_table_1 values(%d, %d, %d, %d, %d, 0, 'binary.%s', 'nchar.%s', %f, %f, %d)''' + % (self.ts + i*10000000, i, i, i, i, i, i, i, i, self.ts + i)) + + tdSql.execute('''insert into regular_table_2 values(%d, %d, %d, %d, %d, 1, 'binary1.%s', 'nchar1.%s', %f, %f, %d)''' + % (self.ts + i*10000000 , 2147483647-i, 9223372036854775807-i, 32767-i, 127-i, + i, i, i, i, 1262304000001 + i)) + tdSql.execute('''insert into regular_table_2 values(%d, %d, %d, %d, %d, true, 'binary2.%s', 'nchar2nchar2.%s', %f, %f, %d)''' + % (self.ts + i*10000000 +2, -2147483647+i, -9223372036854775807+i, -32767+i, -127+i, + i, i, -i, -i, 1577836800001 + i)) + + def regular1_checkall_0(self,sql): + tdLog.info(sql) + tdSql.query(sql) + tdSql.checkData(0,0,'2021-08-27 01:46:40.000') + tdSql.checkData(0,1,0) + tdSql.checkData(0,2,0) + tdSql.checkData(0,3,0) + tdSql.checkData(0,4,0) + tdSql.checkData(0,5,'False') + tdSql.checkData(0,6,'binary.0') + tdSql.checkData(0,7,'nchar.0') + tdSql.checkData(0,8,0) + tdSql.checkData(0,9,0) + tdSql.checkData(0,10,'2021-08-27 01:46:40.000') + + def regular1_checkall_100(self,sql): + tdLog.info(sql) + tdSql.query(sql) + tdSql.checkData(99,0,'2021-09-07 12:46:40.000') + tdSql.checkData(99,1,99) + tdSql.checkData(99,2,99) + tdSql.checkData(99,3,99) + tdSql.checkData(99,4,99) + tdSql.checkData(99,5,'False') + tdSql.checkData(99,6,'binary.99') + tdSql.checkData(99,7,'nchar.99') + tdSql.checkData(99,8,99) + tdSql.checkData(99,9,99) + tdSql.checkData(99,10,'2021-08-27 01:46:40.099') + + def regular_join_checkall_0(self,sql): + self.regular1_checkall_0(sql) + tdSql.checkData(0,11,'2021-08-27 01:46:40.000') + tdSql.checkData(0,12,2147483647) + tdSql.checkData(0,13,9223372036854775807) + tdSql.checkData(0,14,32767) + tdSql.checkData(0,15,127) + tdSql.checkData(0,16,'True') + tdSql.checkData(0,17,'binary1.0') + tdSql.checkData(0,18,'nchar1.0') + tdSql.checkData(0,19,0) + tdSql.checkData(0,20,0) + tdSql.checkData(0,21,'2010-01-01 08:00:00.001') + + def regular_join_checkall_100(self,sql): + self.regular1_checkall_100(sql) + tdSql.checkData(99,11,'2021-09-07 12:46:40.000') + tdSql.checkData(99,12,2147483548) + tdSql.checkData(99,13,9223372036854775708) + tdSql.checkData(99,14,32668) + tdSql.checkData(99,15,28) + tdSql.checkData(99,16,'True') + tdSql.checkData(99,17,'binary1.99') + tdSql.checkData(99,18,'nchar1.99') + tdSql.checkData(99,19,99) + tdSql.checkData(99,20,99) + tdSql.checkData(99,21,'2010-01-01 08:00:00.100') + + def stable1_checkall_0(self,sql): + self.regular1_checkall_0(sql) + + def stable1_checkall_300(self,sql): + tdLog.info(sql) + tdSql.query(sql) + tdSql.checkData(299,0,'2021-09-07 12:46:40.002') + tdSql.checkData(299,1,-2147483548) + tdSql.checkData(299,2,-9223372036854775708) + tdSql.checkData(299,3,-32668) + tdSql.checkData(299,4,-28) + tdSql.checkData(299,5,'True') + tdSql.checkData(299,6,'binary2.99') + tdSql.checkData(299,7,'nchar2nchar2.99') + tdSql.checkData(299,8,-99) + tdSql.checkData(299,9,-99) + tdSql.checkData(299,10,'2010-01-01 08:00:00.100') + + def stable_join_checkall_0(self,sql): + self.regular1_checkall_0(sql) + tdSql.checkData(0,22,'2021-08-27 01:46:40.000') + tdSql.checkData(0,23,0) + tdSql.checkData(0,24,0) + tdSql.checkData(0,25,0) + tdSql.checkData(0,26,0) + tdSql.checkData(0,27,'False') + tdSql.checkData(0,28,'binary.0') + tdSql.checkData(0,29,'nchar.0') + tdSql.checkData(0,30,0) + tdSql.checkData(0,31,0) + tdSql.checkData(0,32,'2021-08-27 01:46:40.000') + + def stable_join_checkall_100(self,sql): + tdSql.checkData(99,0,'2021-09-07 12:46:40.000') + tdSql.checkData(99,1,99) + tdSql.checkData(99,2,99) + tdSql.checkData(99,3,99) + tdSql.checkData(99,4,99) + tdSql.checkData(99,5,'False') + tdSql.checkData(99,6,'binary.99') + tdSql.checkData(99,7,'nchar.99') + tdSql.checkData(99,8,99) + tdSql.checkData(99,9,99) + tdSql.checkData(99,10,'2021-08-27 01:46:40.099') + tdSql.checkData(99,22,'2021-09-07 12:46:40.000') + tdSql.checkData(99,23,99) + tdSql.checkData(99,24,99) + tdSql.checkData(99,25,99) + tdSql.checkData(99,26,99) + tdSql.checkData(99,27,'False') + tdSql.checkData(99,28,'binary.99') + tdSql.checkData(99,29,'nchar.99') + tdSql.checkData(99,30,99) + tdSql.checkData(99,31,99) + tdSql.checkData(99,32,'2021-08-27 01:46:40.099') + + + def run(self): + tdSql.prepare() + # test case for https://jira.taosdata.com:18080/browse/TD-5665 + os.system("rm -rf nestedQuery.py.sql") + startTime = time.time() + + dcDB = self.dropandcreateDB(1) + + # regular column select + q_select= ['ts' , '*' , 'q_int', 'q_bigint' , 'q_bigint' , 'q_smallint' , 'q_tinyint' , 'q_bool' , 'q_binary' , 'q_nchar' ,'q_float' , 'q_double' ,'q_ts '] + + # tag column select + t_select= ['*' , 'loc' ,'t_int', 't_bigint' , 't_bigint' , 't_smallint' , 't_tinyint' , 't_bool' , 't_binary' , 't_nchar' ,'t_float' , 't_double' ,'t_ts '] + + # regular and tag column select + qt_select= q_select + t_select + + # distinct regular column select + dq_select= ['distinct q_int', 'distinct q_bigint' , 'distinct q_smallint' , 'distinct q_tinyint' , + 'distinct q_bool' , 'distinct q_binary' , 'distinct q_nchar' ,'distinct q_float' , 'distinct q_double' ,'distinct q_ts '] + + # distinct tag column select + dt_select= ['distinct loc', 'distinct t_int', 'distinct t_bigint' , 'distinct t_smallint' , 'distinct t_tinyint' , + 'distinct t_bool' , 'distinct t_binary' , 'distinct t_nchar' ,'distinct t_float' , 'distinct t_double' ,'distinct t_ts '] + + # distinct regular and tag column select + dqt_select= dq_select + dt_select + + # special column select + s_r_select= ['_c0', '_C0' ] + s_s_select= ['tbname' , '_c0', '_C0' ] + + # regular column where + q_where = ['ts < now +1s','q_bigint >= -9223372036854775807 and q_bigint <= 9223372036854775807', 'q_int <= 2147483647 and q_int >= -2147483647', + 'q_smallint >= -32767 and q_smallint <= 32767','q_tinyint >= -127 and q_tinyint <= 127','q_float >= -100000 and q_float <= 100000', + 'q_double >= -1000000000 and q_double <= 1000000000', 'q_binary like \'binary%\' or q_binary = \'0\' ' , 'q_nchar like \'nchar%\' or q_nchar = \'0\' ' , + 'q_bool = true or q_bool = false' , 'q_bool in (0 , 1)' , 'q_bool in ( true , false)' , 'q_bool = 0 or q_bool = 1', + 'q_bigint between -9223372036854775807 and 9223372036854775807',' q_int between -2147483647 and 2147483647','q_smallint between -32767 and 32767', + 'q_tinyint between -127 and 127 ','q_float between -100000 and 100000','q_double between -1000000000 and 1000000000'] + #TD-6201 ,'q_bool between 0 and 1' + + # regular column where for test union,join + q_u_where = ['t1.ts < now +1s' , 't2.ts < now +1s','t1.q_bigint >= -9223372036854775807 and t1.q_bigint <= 9223372036854775807 and t2.q_bigint >= -9223372036854775807 and t2.q_bigint <= 9223372036854775807', + 't1.q_int <= 2147483647 and t1.q_int >= -2147483647 and t2.q_int <= 2147483647 and t2.q_int >= -2147483647', + 't1.q_smallint >= -32767 and t1.q_smallint <= 32767 and t2.q_smallint >= -32767 and t2.q_smallint <= 32767', + 't1.q_tinyint >= -127 and t1.q_tinyint <= 127 and t2.q_tinyint >= -127 and t2.q_tinyint <= 127', + 't1.q_float >= -100000 and t1.q_float <= 100000 and t2.q_float >= -100000 and t2.q_float <= 100000', + 't1.q_double >= -1000000000 and t1.q_double <= 1000000000 and t2.q_double >= -1000000000 and t2.q_double <= 1000000000', + 't1.q_binary like \'binary%\' and t2.q_binary like \'binary%\' ' , + 't1.q_nchar like \'nchar%\' and t2.q_nchar like \'nchar%\' ' , + 't1.q_bool in (0 , 1) and t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) and t2.q_bool in ( true , false)' , + 't1.q_bigint between -9223372036854775807 and 9223372036854775807 and t2.q_bigint between -9223372036854775807 and 9223372036854775807', + 't1.q_int between -2147483647 and 2147483647 and t2.q_int between -2147483647 and 2147483647', + 't1.q_smallint between -32767 and 32767 and t2.q_smallint between -32767 and 32767', + 't1.q_tinyint between -127 and 127 and t2.q_tinyint between -127 and 127 ','t1.q_float between -100000 and 100000 and t2.q_float between -100000 and 100000', + 't1.q_double between -1000000000 and 1000000000 and t2.q_double between -1000000000 and 1000000000'] + #TD-6201 ,'t1.q_bool between 0 and 1 or t2.q_bool between 0 and 1'] + #'t1.q_bool = true and t1.q_bool = false and t2.q_bool = true and t2.q_bool = false' , 't1.q_bool = 0 and t1.q_bool = 1 and t2.q_bool = 0 and t2.q_bool = 1' , + + q_u_or_where = ['t1.q_binary like \'binary%\' or t1.q_binary = \'0\' or t2.q_binary like \'binary%\' or t2.q_binary = \'0\' ' , + 't1.q_nchar like \'nchar%\' or t1.q_nchar = \'0\' or t2.q_nchar like \'nchar%\' or t2.q_nchar = \'0\' ' , 't1.q_bool = true or t1.q_bool = false or t2.q_bool = true or t2.q_bool = false' , + 't1.q_bool in (0 , 1) or t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) or t2.q_bool in ( true , false)' , 't1.q_bool = 0 or t1.q_bool = 1 or t2.q_bool = 0 or t2.q_bool = 1' , + 't1.q_bigint between -9223372036854775807 and 9223372036854775807 or t2.q_bigint between -9223372036854775807 and 9223372036854775807', + 't1.q_int between -2147483647 and 2147483647 or t2.q_int between -2147483647 and 2147483647', + 't1.q_smallint between -32767 and 32767 or t2.q_smallint between -32767 and 32767', + 't1.q_tinyint between -127 and 127 or t2.q_tinyint between -127 and 127 ','t1.q_float between -100000 and 100000 or t2.q_float between -100000 and 100000', + 't1.q_double between -1000000000 and 1000000000 or t2.q_double between -1000000000 and 1000000000'] + + # tag column where + t_where = ['ts < now +1s','t_bigint >= -9223372036854775807 and t_bigint <= 9223372036854775807','t_int <= 2147483647 and t_int >= -2147483647', + 't_smallint >= -32767 and t_smallint <= 32767','q_tinyint >= -127 and t_tinyint <= 127','t_float >= -100000 and t_float <= 100000', + 't_double >= -1000000000 and t_double <= 1000000000', 't_binary like \'binary%\' or t_binary = \'0\' ' , 't_nchar like \'nchar%\' or t_nchar = \'0\'' , + 't_bool = true or t_bool = false' , 't_bool in (0 , 1)' , 't_bool in ( true , false)' , 't_bool = 0 or t_bool = 1', + 't_bigint between -9223372036854775807 and 9223372036854775807',' t_int between -2147483647 and 2147483647','t_smallint between -32767 and 32767', + 't_tinyint between -127 and 127 ','t_float between -100000 and 100000','t_double between -1000000000 and 1000000000'] + #TD-6201,'t_bool between 0 and 1' + + # tag column where for test union,join | this is not support + t_u_where = ['t1.ts < now +1s' , 't2.ts < now +1s','t1.t_bigint >= -9223372036854775807 and t1.t_bigint <= 9223372036854775807 and t2.t_bigint >= -9223372036854775807 and t2.t_bigint <= 9223372036854775807', + 't1.t_int <= 2147483647 and t1.t_int >= -2147483647 and t2.t_int <= 2147483647 and t2.t_int >= -2147483647', + 't1.t_smallint >= -32767 and t1.t_smallint <= 32767 and t2.t_smallint >= -32767 and t2.t_smallint <= 32767', + 't1.t_tinyint >= -127 and t1.t_tinyint <= 127 and t2.t_tinyint >= -127 and t2.t_tinyint <= 127', + 't1.t_float >= -100000 and t1.t_float <= 100000 and t2.t_float >= -100000 and t2.t_float <= 100000', + 't1.t_double >= -1000000000 and t1.t_double <= 1000000000 and t2.t_double >= -1000000000 and t2.t_double <= 1000000000', + 't1.t_binary like \'binary%\' or t1.t_binary = \'0\' or t2.t_binary like \'binary%\' or t2.t_binary = \'0\' ' , + 't1.t_nchar like \'nchar%\' or t1.t_nchar = \'0\' or t2.t_nchar like \'nchar%\' or t2.t_nchar = \'0\' ' , 't1.t_bool = true or t1.t_bool = false or t2.t_bool = true or t2.t_bool = false' , + 't1.t_bool in (0 , 1) and t2.t_bool in (0 , 1)' , 't1.t_bool in ( true , false) and t2.t_bool in ( true , false)' , 't1.t_bool = 0 or t1.t_bool = 1 or t2.t_bool = 0 or t2.t_bool = 1', + 't1.t_bigint between -9223372036854775807 and 9223372036854775807 and t2.t_bigint between -9223372036854775807 and 9223372036854775807', + 't1.t_int between -2147483647 and 2147483647 and t2.t_int between -2147483647 and 2147483647', + 't1.t_smallint between -32767 and 32767 and t2.t_smallint between -32767 and 32767', + 't1.t_tinyint between -127 and 127 and t2.t_tinyint between -127 and 127 ','t1.t_float between -100000 and 100000 and t2.t_float between -100000 and 100000', + 't1.t_double between -1000000000 and 1000000000 and t2.t_double between -1000000000 and 1000000000'] + #TD-6201,'t1.t_bool between 0 and 1 or t2.q_bool between 0 and 1'] + + t_u_or_where = ['t1.t_binary like \'binary%\' or t1.t_binary = \'0\' or t2.t_binary like \'binary%\' or t2.t_binary = \'0\' ' , + 't1.t_nchar like \'nchar%\' or t1.t_nchar = \'0\' or t2.t_nchar like \'nchar%\' or t2.t_nchar = \'0\' ' , 't1.t_bool = true or t1.t_bool = false or t2.t_bool = true or t2.t_bool = false' , + 't1.t_bool in (0 , 1) or t2.t_bool in (0 , 1)' , 't1.t_bool in ( true , false) or t2.t_bool in ( true , false)' , 't1.t_bool = 0 or t1.t_bool = 1 or t2.t_bool = 0 or t2.t_bool = 1', + 't1.t_bigint between -9223372036854775807 and 9223372036854775807 or t2.t_bigint between -9223372036854775807 and 9223372036854775807', + 't1.t_int between -2147483647 and 2147483647 or t2.t_int between -2147483647 and 2147483647', + 't1.t_smallint between -32767 and 32767 or t2.t_smallint between -32767 and 32767', + 't1.t_tinyint between -127 and 127 or t2.t_tinyint between -127 and 127 ','t1.t_float between -100000 and 100000 or t2.t_float between -100000 and 100000', + 't1.t_double between -1000000000 and 1000000000 or t2.t_double between -1000000000 and 1000000000'] + + # regular and tag column where + qt_where = q_where + t_where + qt_u_where = q_u_where + t_u_where + # now,qt_u_or_where is not support + qt_u_or_where = q_u_or_where + t_u_or_where + + # tag column where for test super join | this is support , 't1.t_bool = t2.t_bool ' ？？？ + t_join_where = ['t1.t_bigint = t2.t_bigint ', 't1.t_int = t2.t_int ', 't1.t_smallint = t2.t_smallint ', 't1.t_tinyint = t2.t_tinyint ', + 't1.t_float = t2.t_float ', 't1.t_double = t2.t_double ', 't1.t_binary = t2.t_binary ' , 't1.t_nchar = t2.t_nchar ' ] + + # session && fill + session_where = ['session(ts,10a)' , 'session(ts,10s)', 'session(ts,10m)' , 'session(ts,10h)','session(ts,10d)' , 'session(ts,10w)'] + session_u_where = ['session(t1.ts,10a)' , 'session(t1.ts,10s)', 'session(t1.ts,10m)' , 'session(t1.ts,10h)','session(t1.ts,10d)' , 'session(t1.ts,10w)', + 'session(t2.ts,10a)' , 'session(t2.ts,10s)', 'session(t2.ts,10m)' , 'session(t2.ts,10h)','session(t2.ts,10d)' , 'session(t2.ts,10w)'] + + fill_where = ['FILL(NONE)','FILL(PREV)','FILL(NULL)','FILL(LINEAR)','FILL(NEXT)','FILL(VALUE, 1.23)'] + + state_window = ['STATE_WINDOW(q_tinyint)','STATE_WINDOW(q_bigint)','STATE_WINDOW(q_int)','STATE_WINDOW(q_bool)','STATE_WINDOW(q_smallint)'] + state_u_window = ['STATE_WINDOW(t1.q_tinyint)','STATE_WINDOW(t1.q_bigint)','STATE_WINDOW(t1.q_int)','STATE_WINDOW(t1.q_bool)','STATE_WINDOW(t1.q_smallint)', + 'STATE_WINDOW(t2.q_tinyint)','STATE_WINDOW(t2.q_bigint)','STATE_WINDOW(t2.q_int)','STATE_WINDOW(t2.q_bool)','STATE_WINDOW(t2.q_smallint)'] + + # order by where + order_where = ['order by ts' , 'order by ts asc'] + order_u_where = ['order by t1.ts' , 'order by t1.ts asc' , 'order by t2.ts' , 'order by t2.ts asc'] + order_desc_where = ['order by ts' , 'order by ts asc' , 'order by ts desc' ] + orders_desc_where = ['order by ts' , 'order by ts asc' , 'order by ts desc' , 'order by loc' , 'order by loc asc' , 'order by loc desc'] + + # group by where,not include null-tag + group_where = ['group by tbname , loc' , 'group by tbname', 'group by tbname, t_bigint', 'group by tbname,t_int', 'group by tbname, t_smallint', 'group by tbname,t_tinyint', + 'group by tbname,t_float', 'group by tbname,t_double' , 'group by tbname,t_binary', 'group by tbname,t_nchar', 'group by tbname,t_bool' ,'group by tbname ,loc ,t_bigint', + 'group by tbname,t_binary ,t_nchar ,t_bool' , 'group by tbname,t_int ,t_smallint ,t_tinyint' , 'group by tbname,t_float ,t_double ' ] + having_support = ['having count(q_int) > 0','having count(q_bigint) > 0','having count(q_smallint) > 0','having count(q_tinyint) > 0','having count(q_float) > 0','having count(q_double) > 0','having count(q_bool) > 0', + 'having avg(q_int) > 0','having avg(q_bigint) > 0','having avg(q_smallint) > 0','having avg(q_tinyint) > 0','having avg(q_float) > 0','having avg(q_double) > 0', + 'having sum(q_int) > 0','having sum(q_bigint) > 0','having sum(q_smallint) > 0','having sum(q_tinyint) > 0','having sum(q_float) > 0','having sum(q_double) > 0', + 'having STDDEV(q_int) > 0','having STDDEV(q_bigint) > 0','having STDDEV(q_smallint) > 0','having STDDEV(q_tinyint) > 0','having STDDEV(q_float) > 0','having STDDEV(q_double) > 0', + 'having TWA(q_int) > 0','having TWA(q_bigint) > 0','having TWA(q_smallint) > 0','having TWA(q_tinyint) > 0','having TWA(q_float) > 0','having TWA(q_double) > 0', + 'having IRATE(q_int) > 0','having IRATE(q_bigint) > 0','having IRATE(q_smallint) > 0','having IRATE(q_tinyint) > 0','having IRATE(q_float) > 0','having IRATE(q_double) > 0', + 'having MIN(q_int) > 0','having MIN(q_bigint) > 0','having MIN(q_smallint) > 0','having MIN(q_tinyint) > 0','having MIN(q_float) > 0','having MIN(q_double) > 0', + 'having MAX(q_int) > 0','having MAX(q_bigint) > 0','having MAX(q_smallint) > 0','having MAX(q_tinyint) > 0','having MAX(q_float) > 0','having MAX(q_double) > 0', + 'having FIRST(q_int) > 0','having FIRST(q_bigint) > 0','having FIRST(q_smallint) > 0','having FIRST(q_tinyint) > 0','having FIRST(q_float) > 0','having FIRST(q_double) > 0', + 'having LAST(q_int) > 0','having LAST(q_bigint) > 0','having LAST(q_smallint) > 0','having LAST(q_tinyint) > 0','having LAST(q_float) > 0','having LAST(q_double) > 0', + 'having APERCENTILE(q_int,10) > 0','having APERCENTILE(q_bigint,10) > 0','having APERCENTILE(q_smallint,10) > 0','having APERCENTILE(q_tinyint,10) > 0','having APERCENTILE(q_float,10) > 0','having APERCENTILE(q_double,10) > 0'] + having_not_support = ['having TOP(q_int,10) > 0','having TOP(q_bigint,10) > 0','having TOP(q_smallint,10) > 0','having TOP(q_tinyint,10) > 0','having TOP(q_float,10) > 0','having TOP(q_double,10) > 0','having TOP(q_bool,10) > 0', + 'having BOTTOM(q_int,10) > 0','having BOTTOM(q_bigint,10) > 0','having BOTTOM(q_smallint,10) > 0','having BOTTOM(q_tinyint,10) > 0','having BOTTOM(q_float,10) > 0','having BOTTOM(q_double,10) > 0','having BOTTOM(q_bool,10) > 0', + 'having LEASTSQUARES(q_int) > 0','having LEASTSQUARES(q_bigint) > 0','having LEASTSQUARES(q_smallint) > 0','having LEASTSQUARES(q_tinyint) > 0','having LEASTSQUARES(q_float) > 0','having LEASTSQUARES(q_double) > 0','having LEASTSQUARES(q_bool) > 0', + 'having FIRST(q_bool) > 0','having IRATE(q_bool) > 0','having PERCENTILE(q_bool,10) > 0','having avg(q_bool) > 0','having LAST_ROW(q_bool) > 0','having sum(q_bool) > 0','having STDDEV(q_bool) > 0','having APERCENTILE(q_bool,10) > 0','having TWA(q_bool) > 0','having LAST(q_bool) > 0', + 'having PERCENTILE(q_int,10) > 0','having PERCENTILE(q_bigint,10) > 0','having PERCENTILE(q_smallint,10) > 0','having PERCENTILE(q_tinyint,10) > 0','having PERCENTILE(q_float,10) > 0','having PERCENTILE(q_double,10) > 0'] + having_tagnot_support = ['having LAST_ROW(q_int) > 0','having LAST_ROW(q_bigint) > 0','having LAST_ROW(q_smallint) > 0','having LAST_ROW(q_tinyint) > 0','having LAST_ROW(q_float) > 0','having LAST_ROW(q_double) > 0'] + + # limit offset where + limit_where = ['limit 1 offset 1' , 'limit 1' , 'limit 2 offset 1' , 'limit 2', 'limit 12 offset 1' , 'limit 20', 'limit 20 offset 10' , 'limit 200'] + limit1_where = ['limit 1 offset 1' , 'limit 1' ] + limit_u_where = ['limit 100 offset 10' , 'limit 50' , 'limit 100' , 'limit 10' ] + + # slimit soffset where + slimit_where = ['slimit 1 soffset 1' , 'slimit 1' , 'slimit 2 soffset 1' , 'slimit 2'] + slimit1_where = ['slimit 2 soffset 1' , 'slimit 1' ] + + # aggregate function include [all:count(*)\avg\sum\stddev ||regualr:twa\irate\leastsquares ||group by tbname:twa\irate\] + # select function include [all: min\max\first(*)\last(*)\top\bottom\apercentile\last_row(*)(not with interval)\interp(*)(FILL) ||regualr: percentile] + # calculation function include [all:spread\+-*/ ||regualr:diff\derivative ||group by tbname:diff\derivative\] + # **_ns_** express is not support stable, therefore, separated from regular tables + # calc_select_all calc_select_regular calc_select_in_ts calc_select_fill calc_select_not_interval + # calc_aggregate_all calc_aggregate_regular calc_aggregate_groupbytbname + # calc_calculate_all calc_calculate_regular calc_calculate_groupbytbname + + # calc_select_all calc_select_regular calc_select_in_ts calc_select_fill calc_select_not_interval + # select function include [all: min\max\first(*)\last(*)\top\bottom\apercentile\last_row(*)(not with interval)\interp(*)(FILL) ||regualr: percentile] + + calc_select_all = ['bottom(q_int,20)' , 'bottom(q_bigint,20)' , 'bottom(q_smallint,20)' , 'bottom(q_tinyint,20)' ,'bottom(q_float,20)' , 'bottom(q_double,20)' , + 'top(q_int,20)' , 'top(q_bigint,20)' , 'top(q_smallint,20)' ,'top(q_tinyint,20)' ,'top(q_float,20)' ,'top(q_double,20)' , + 'first(q_int)' , 'first(q_bigint)' , 'first(q_smallint)' , 'first(q_tinyint)' , 'first(q_float)' ,'first(q_double)' ,'first(q_binary)' ,'first(q_nchar)' ,'first(q_bool)' ,'first(q_ts)' , + 'last(q_int)' , 'last(q_bigint)' , 'last(q_smallint)' , 'last(q_tinyint)' , 'last(q_float)' ,'last(q_double)' , 'last(q_binary)' ,'last(q_nchar)' ,'last(q_bool)' ,'last(q_ts)' , + 'min(q_int)' , 'min(q_bigint)' , 'min(q_smallint)' , 'min(q_tinyint)' , 'min(q_float)' ,'min(q_double)' , + 'max(q_int)' , 'max(q_bigint)' , 'max(q_smallint)' , 'max(q_tinyint)' ,'max(q_float)' ,'max(q_double)' , + 'apercentile(q_int,20)' , 'apercentile(q_bigint,20)' ,'apercentile(q_smallint,20)' ,'apercentile(q_tinyint,20)' ,'apercentile(q_float,20)' ,'apercentile(q_double,20)' , + 'last_row(q_int)' , 'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , + 'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)'] + + calc_select_in_ts = ['bottom(q_int,20)' , 'bottom(q_bigint,20)' , 'bottom(q_smallint,20)' , 'bottom(q_tinyint,20)' ,'bottom(q_float,20)' , 'bottom(q_double,20)' , + 'top(q_int,20)' , 'top(q_bigint,20)' , 'top(q_smallint,20)' ,'top(q_tinyint,20)' ,'top(q_float,20)' ,'top(q_double,20)' , + 'first(q_int)' , 'first(q_bigint)' , 'first(q_smallint)' , 'first(q_tinyint)' , 'first(q_float)' ,'first(q_double)' ,'first(q_binary)' ,'first(q_nchar)' ,'first(q_bool)' ,'first(q_ts)' , + 'last(q_int)' , 'last(q_bigint)' , 'last(q_smallint)' , 'last(q_tinyint)' , 'last(q_float)' ,'last(q_double)' , 'last(q_binary)' ,'last(q_nchar)' ,'last(q_bool)' ,'last(q_ts)' ] + + calc_select_in = ['min(q_int)' , 'min(q_bigint)' , 'min(q_smallint)' , 'min(q_tinyint)' , 'min(q_float)' ,'min(q_double)' , + 'max(q_int)' , 'max(q_bigint)' , 'max(q_smallint)' , 'max(q_tinyint)' ,'max(q_float)' ,'max(q_double)' , + 'apercentile(q_int,20)' , 'apercentile(q_bigint,20)' ,'apercentile(q_smallint,20)' ,'apercentile(q_tinyint,20)' ,'apercentile(q_float,20)' ,'apercentile(q_double,20)' , + 'last_row(q_int)' , 'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , + 'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)'] + + + calc_select_regular = [ 'PERCENTILE(q_int,10)' ,'PERCENTILE(q_bigint,20)' , 'PERCENTILE(q_smallint,30)' ,'PERCENTILE(q_tinyint,40)' ,'PERCENTILE(q_float,50)' ,'PERCENTILE(q_double,60)'] + + + calc_select_fill = ['INTERP(q_bool)' ,'INTERP(q_binary)' ,'INTERP(q_nchar)' ,'INTERP(q_ts)', 'INTERP(q_int)' ,'INTERP(*)' ,'INTERP(q_bigint)' ,'INTERP(q_smallint)' ,'INTERP(q_tinyint)', 'INTERP(q_float)' ,'INTERP(q_double)'] + interp_where = ['ts = now' , 'ts = \'2020-09-13 20:26:40.000\'' , 'ts = \'2020-09-13 20:26:40.009\'' ,'tbname in (\'table_1\') and ts = now' ,'tbname in (\'table_0\' ,\'table_1\',\'table_2\',\'table_3\',\'table_4\',\'table_5\') and ts = \'2020-09-13 20:26:40.000\'','tbname like \'table%\' and ts = \'2020-09-13 20:26:40.002\''] + + #two table join + calc_select_in_ts_j = ['bottom(t1.q_int,20)' , 'bottom(t1.q_bigint,20)' , 'bottom(t1.q_smallint,20)' , 'bottom(t1.q_tinyint,20)' ,'bottom(t1.q_float,20)' , 'bottom(t1.q_double,20)' , + 'top(t1.q_int,20)' , 'top(t1.q_bigint,20)' , 'top(t1.q_smallint,20)' ,'top(t1.q_tinyint,20)' ,'top(t1.q_float,20)' ,'top(t1.q_double,20)' , + 'first(t1.q_int)' , 'first(t1.q_bigint)' , 'first(t1.q_smallint)' , 'first(t1.q_tinyint)' , 'first(t1.q_float)' ,'first(t1.q_double)' ,'first(t1.q_binary)' ,'first(t1.q_nchar)' ,'first(t1.q_bool)' ,'first(t1.q_ts)' , + 'last(t1.q_int)' , 'last(t1.q_bigint)' , 'last(t1.q_smallint)' , 'last(t1.q_tinyint)' , 'last(t1.q_float)' ,'last(t1.q_double)' , 'last(t1.q_binary)' ,'last(t1.q_nchar)' ,'last(t1.q_bool)' ,'last(t1.q_ts)' , + 'bottom(t2.q_int,20)' , 'bottom(t2.q_bigint,20)' , 'bottom(t2.q_smallint,20)' , 'bottom(t2.q_tinyint,20)' ,'bottom(t2.q_float,20)' , 'bottom(t2.q_double,20)' , + 'top(t2.q_int,20)' , 'top(t2.q_bigint,20)' , 'top(t2.q_smallint,20)' ,'top(t2.q_tinyint,20)' ,'top(t2.q_float,20)' ,'top(t2.q_double,20)' , + 'first(t2.q_int)' , 'first(t2.q_bigint)' , 'first(t2.q_smallint)' , 'first(t2.q_tinyint)' , 'first(t2.q_float)' ,'first(t2.q_double)' ,'first(t2.q_binary)' ,'first(t2.q_nchar)' ,'first(t2.q_bool)' ,'first(t2.q_ts)' , + 'last(t2.q_int)' , 'last(t2.q_bigint)' , 'last(t2.q_smallint)' , 'last(t2.q_tinyint)' , 'last(t2.q_float)' ,'last(t2.q_double)' , 'last(t2.q_binary)' ,'last(t2.q_nchar)' ,'last(t2.q_bool)' ,'last(t2.q_ts)'] + + calc_select_in_j = ['min(t1.q_int)' , 'min(t1.q_bigint)' , 'min(t1.q_smallint)' , 'min(t1.q_tinyint)' , 'min(t1.q_float)' ,'min(t1.q_double)' , + 'max(t1.q_int)' , 'max(t1.q_bigint)' , 'max(t1.q_smallint)' , 'max(t1.q_tinyint)' ,'max(t1.q_float)' ,'max(t1.q_double)' , + 'apercentile(t1.q_int,20)' , 'apercentile(t1.q_bigint,20)' ,'apercentile(t1.q_smallint,20)' ,'apercentile(t1.q_tinyint,20)' ,'apercentile(t1.q_float,20)' ,'apercentile(t1.q_double,20)' , + 'last_row(t1.q_int)' , 'last_row(t1.q_bigint)' , 'last_row(t1.q_smallint)' , 'last_row(t1.q_tinyint)' , 'last_row(t1.q_float)' , + 'last_row(t1.q_double)' , 'last_row(t1.q_bool)' ,'last_row(t1.q_binary)' ,'last_row(t1.q_nchar)' ,'last_row(t1.q_ts)' , + 'min(t2.q_int)' , 'min(t2.q_bigint)' , 'min(t2.q_smallint)' , 'min(t2.q_tinyint)' , 'min(t2.q_float)' ,'min(t2.q_double)' , + 'max(t2.q_int)' , 'max(t2.q_bigint)' , 'max(t2.q_smallint)' , 'max(t2.q_tinyint)' ,'max(t2.q_float)' ,'max(t2.q_double)' , + 'apercentile(t2.q_int,20)' , 'apercentile(t2.q_bigint,20)' ,'apercentile(t2.q_smallint,20)' ,'apercentile(t2.q_tinyint,20)' ,'apercentile(t2.q_float,20)' ,'apercentile(t2.q_double,20)' , + 'last_row(t2.q_int)' , 'last_row(t2.q_bigint)' , 'last_row(t2.q_smallint)' , 'last_row(t2.q_tinyint)' , 'last_row(t2.q_float)' , + 'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)'] + + calc_select_all_j = calc_select_in_ts_j + calc_select_in_j + + calc_select_regular_j = [ 'PERCENTILE(t1.q_int,10)' ,'PERCENTILE(t1.q_bigint,20)' , 'PERCENTILE(t1.q_smallint,30)' ,'PERCENTILE(t1.q_tinyint,40)' ,'PERCENTILE(t1.q_float,50)' ,'PERCENTILE(t1.q_double,60)' , + 'PERCENTILE(t2.q_int,10)' ,'PERCENTILE(t2.q_bigint,20)' , 'PERCENTILE(t2.q_smallint,30)' ,'PERCENTILE(t2.q_tinyint,40)' ,'PERCENTILE(t2.q_float,50)' ,'PERCENTILE(t2.q_double,60)'] + + + calc_select_fill_j = ['INTERP(t1.q_bool)' ,'INTERP(t1.q_binary)' ,'INTERP(t1.q_nchar)' ,'INTERP(t1.q_ts)', 'INTERP(t1.q_int)' ,'INTERP(t1.*)' ,'INTERP(t1.q_bigint)' ,'INTERP(t1.q_smallint)' ,'INTERP(t1.q_tinyint)', 'INTERP(t1.q_float)' ,'INTERP(t1.q_double)' , + 'INTERP(t2.q_bool)' ,'INTERP(t2.q_binary)' ,'INTERP(t2.q_nchar)' ,'INTERP(t2.q_ts)', 'INTERP(t2.q_int)' ,'INTERP(t2.*)' ,'INTERP(t2.q_bigint)' ,'INTERP(t2.q_smallint)' ,'INTERP(t2.q_tinyint)', 'INTERP(t2.q_float)' ,'INTERP(t2.q_double)'] + interp_where_j = ['t1.ts = now' , 't1.ts = \'2020-09-13 20:26:40.000\'' , 't1.ts = \'2020-09-13 20:26:40.009\'' ,'t2.ts = now' , 't2.ts = \'2020-09-13 20:26:40.000\'' , 't2.ts = \'2020-09-13 20:26:40.009\'' , + 't1.tbname in (\'table_1\') and t1.ts = now' ,'t1.tbname in (\'table_0\' ,\'table_1\',\'table_2\',\'table_3\',\'table_4\',\'table_5\') and t1.ts = \'2020-09-13 20:26:40.000\'','t1.tbname like \'table%\' and t1.ts = \'2020-09-13 20:26:40.002\'', + 't2.tbname in (\'table_1\') and t2.ts = now' ,'t2.tbname in (\'table_0\' ,\'table_1\',\'table_2\',\'table_3\',\'table_4\',\'table_5\') and t2.ts = \'2020-09-13 20:26:40.000\'','t2.tbname like \'table%\' and t2.ts = \'2020-09-13 20:26:40.002\''] + + # calc_aggregate_all calc_aggregate_regular calc_aggregate_groupbytbname APERCENTILE\PERCENTILE + # aggregate function include [all:count(*)\avg\sum\stddev ||regualr:twa\irate\leastsquares ||group by tbname:twa\irate\] + calc_aggregate_all = ['count(*)' , 'count(q_int)' ,'count(q_bigint)' , 'count(q_smallint)' ,'count(q_tinyint)' ,'count(q_float)' , + 'count(q_double)' ,'count(q_binary)' ,'count(q_nchar)' ,'count(q_bool)' ,'count(q_ts)' , + 'avg(q_int)' ,'avg(q_bigint)' , 'avg(q_smallint)' ,'avg(q_tinyint)' ,'avg(q_float)' ,'avg(q_double)' , + 'sum(q_int)' ,'sum(q_bigint)' , 'sum(q_smallint)' ,'sum(q_tinyint)' ,'sum(q_float)' ,'sum(q_double)' , + 'STDDEV(q_int)' ,'STDDEV(q_bigint)' , 'STDDEV(q_smallint)' ,'STDDEV(q_tinyint)' ,'STDDEV(q_float)' ,'STDDEV(q_double)', + 'APERCENTILE(q_int,10)' ,'APERCENTILE(q_bigint,20)' , 'APERCENTILE(q_smallint,30)' ,'APERCENTILE(q_tinyint,40)' ,'APERCENTILE(q_float,50)' ,'APERCENTILE(q_double,60)'] + + calc_aggregate_regular = ['twa(q_int)' ,'twa(q_bigint)' , 'twa(q_smallint)' ,'twa(q_tinyint)' ,'twa (q_float)' ,'twa(q_double)' , + 'IRATE(q_int)' ,'IRATE(q_bigint)' , 'IRATE(q_smallint)' ,'IRATE(q_tinyint)' ,'IRATE (q_float)' ,'IRATE(q_double)' , + 'LEASTSQUARES(q_int,15,3)' , 'LEASTSQUARES(q_bigint,10,1)' , 'LEASTSQUARES(q_smallint,20,3)' ,'LEASTSQUARES(q_tinyint,10,4)' ,'LEASTSQUARES(q_float,6,4)' ,'LEASTSQUARES(q_double,3,1)' , + 'PERCENTILE(q_int,10)' ,'PERCENTILE(q_bigint,20)' , 'PERCENTILE(q_smallint,30)' ,'PERCENTILE(q_tinyint,40)' ,'PERCENTILE(q_float,50)' ,'PERCENTILE(q_double,60)'] + + calc_aggregate_groupbytbname = ['twa(q_int)' ,'twa(q_bigint)' , 'twa(q_smallint)' ,'twa(q_tinyint)' ,'twa (q_float)' ,'twa(q_double)' , + 'IRATE(q_int)' ,'IRATE(q_bigint)' , 'IRATE(q_smallint)' ,'IRATE(q_tinyint)' ,'IRATE (q_float)' ,'IRATE(q_double)' ] + + #two table join + calc_aggregate_all_j = ['count(t1.*)' , 'count(t1.q_int)' ,'count(t1.q_bigint)' , 'count(t1.q_smallint)' ,'count(t1.q_tinyint)' ,'count(t1.q_float)' , + 'count(t1.q_double)' ,'count(t1.q_binary)' ,'count(t1.q_nchar)' ,'count(t1.q_bool)' ,'count(t1.q_ts)' , + 'avg(t1.q_int)' ,'avg(t1.q_bigint)' , 'avg(t1.q_smallint)' ,'avg(t1.q_tinyint)' ,'avg(t1.q_float)' ,'avg(t1.q_double)' , + 'sum(t1.q_int)' ,'sum(t1.q_bigint)' , 'sum(t1.q_smallint)' ,'sum(t1.q_tinyint)' ,'sum(t1.q_float)' ,'sum(t1.q_double)' , + 'STDDEV(t1.q_int)' ,'STDDEV(t1.q_bigint)' , 'STDDEV(t1.q_smallint)' ,'STDDEV(t1.q_tinyint)' ,'STDDEV(t1.q_float)' ,'STDDEV(t1.q_double)', + 'APERCENTILE(t1.q_int,10)' ,'APERCENTILE(t1.q_bigint,20)' , 'APERCENTILE(t1.q_smallint,30)' ,'APERCENTILE(t1.q_tinyint,40)' ,'APERCENTILE(t1.q_float,50)' ,'APERCENTILE(t1.q_double,60)' , + 'count(t2.*)' , 'count(t2.q_int)' ,'count(t2.q_bigint)' , 'count(t2.q_smallint)' ,'count(t2.q_tinyint)' ,'count(t2.q_float)' , + 'count(t2.q_double)' ,'count(t2.q_binary)' ,'count(t2.q_nchar)' ,'count(t2.q_bool)' ,'count(t2.q_ts)' , + 'avg(t2.q_int)' ,'avg(t2.q_bigint)' , 'avg(t2.q_smallint)' ,'avg(t2.q_tinyint)' ,'avg(t2.q_float)' ,'avg(t2.q_double)' , + 'sum(t2.q_int)' ,'sum(t2.q_bigint)' , 'sum(t2.q_smallint)' ,'sum(t2.q_tinyint)' ,'sum(t2.q_float)' ,'sum(t2.q_double)' , + 'STDDEV(t2.q_int)' ,'STDDEV(t2.q_bigint)' , 'STDDEV(t2.q_smallint)' ,'STDDEV(t2.q_tinyint)' ,'STDDEV(t2.q_float)' ,'STDDEV(t2.q_double)', + 'APERCENTILE(t2.q_int,10)' ,'APERCENTILE(t2.q_bigint,20)' , 'APERCENTILE(t2.q_smallint,30)' ,'APERCENTILE(t2.q_tinyint,40)' ,'APERCENTILE(t2.q_float,50)' ,'APERCENTILE(t2.q_double,60)'] + + calc_aggregate_regular_j = ['twa(t1.q_int)' ,'twa(t1.q_bigint)' , 'twa(t1.q_smallint)' ,'twa(t1.q_tinyint)' ,'twa (t1.q_float)' ,'twa(t1.q_double)' , + 'IRATE(t1.q_int)' ,'IRATE(t1.q_bigint)' , 'IRATE(t1.q_smallint)' ,'IRATE(t1.q_tinyint)' ,'IRATE (t1.q_float)' ,'IRATE(t1.q_double)' , + 'LEASTSQUARES(t1.q_int,15,3)' , 'LEASTSQUARES(t1.q_bigint,10,1)' , 'LEASTSQUARES(t1.q_smallint,20,3)' ,'LEASTSQUARES(t1.q_tinyint,10,4)' ,'LEASTSQUARES(t1.q_float,6,4)' ,'LEASTSQUARES(t1.q_double,3,1)' , + 'PERCENTILE(t1.q_int,10)' ,'PERCENTILE(t1.q_bigint,20)' , 'PERCENTILE(t1.q_smallint,30)' ,'PERCENTILE(t1.q_tinyint,40)' ,'PERCENTILE(t1.q_float,50)' ,'PERCENTILE(t1.q_double,60)' , + 'twa(t2.q_int)' ,'twa(t2.q_bigint)' , 'twa(t2.q_smallint)' ,'twa(t2.q_tinyint)' ,'twa (t2.q_float)' ,'twa(t2.q_double)' , + 'IRATE(t2.q_int)' ,'IRATE(t2.q_bigint)' , 'IRATE(t2.q_smallint)' ,'IRATE(t2.q_tinyint)' ,'IRATE (t2.q_float)' ,'IRATE(t2.q_double)', + 'LEASTSQUARES(t2.q_int,15,3)' , 'LEASTSQUARES(t2.q_bigint,10,1)' , 'LEASTSQUARES(t2.q_smallint,20,3)' ,'LEASTSQUARES(t2.q_tinyint,10,4)' ,'LEASTSQUARES(t2.q_float,6,4)' ,'LEASTSQUARES(t2.q_double,3,1)' , + 'PERCENTILE(t2.q_int,10)' ,'PERCENTILE(t2.q_bigint,20)' , 'PERCENTILE(t2.q_smallint,30)' ,'PERCENTILE(t2.q_tinyint,40)' ,'PERCENTILE(t2.q_float,50)' ,'PERCENTILE(t2.q_double,60)'] + + calc_aggregate_groupbytbname_j = ['twa(t1.q_int)' ,'twa(t1.q_bigint)' , 'twa(t1.q_smallint)' ,'twa(t1.q_tinyint)' ,'twa (t1.q_float)' ,'twa(t1.q_double)' , + 'IRATE(t1.q_int)' ,'IRATE(t1.q_bigint)' , 'IRATE(t1.q_smallint)' ,'IRATE(t1.q_tinyint)' ,'IRATE (t1.q_float)' ,'IRATE(t1.q_double)' , + 'twa(t2.q_int)' ,'twa(t2.q_bigint)' , 'twa(t2.q_smallint)' ,'twa(t2.q_tinyint)' ,'twa (t2.q_float)' ,'twa(t2.q_double)' , + 'IRATE(t2.q_int)' ,'IRATE(t2.q_bigint)' , 'IRATE(t2.q_smallint)' ,'IRATE(t2.q_tinyint)' ,'IRATE (t2.q_float)' ,'IRATE(t2.q_double)' ] + + # calc_calculate_all calc_calculate_regular calc_calculate_groupbytbname + # calculation function include [all:spread\+-*/ ||regualr:diff\derivative ||group by tbname:diff\derivative\] + calc_calculate_all = ['SPREAD(ts)' , 'SPREAD(q_ts)' , 'SPREAD(q_int)' ,'SPREAD(q_bigint)' , 'SPREAD(q_smallint)' ,'SPREAD(q_tinyint)' ,'SPREAD(q_float)' ,'SPREAD(q_double)' , + '(SPREAD(q_int) + SPREAD(q_bigint))' , '(SPREAD(q_smallint) - SPREAD(q_float))', '(SPREAD(q_double) * SPREAD(q_tinyint))' , '(SPREAD(q_double) / SPREAD(q_float))'] + calc_calculate_regular = ['DIFF(q_int)' ,'DIFF(q_bigint)' , 'DIFF(q_smallint)' ,'DIFF(q_tinyint)' ,'DIFF(q_float)' ,'DIFF(q_double)' , + 'DERIVATIVE(q_int,15s,0)' , 'DERIVATIVE(q_bigint,10s,1)' , 'DERIVATIVE(q_smallint,20s,0)' ,'DERIVATIVE(q_tinyint,10s,1)' ,'DERIVATIVE(q_float,6s,0)' ,'DERIVATIVE(q_double,3s,1)' ] + calc_calculate_groupbytbname = calc_calculate_regular + + #two table join + calc_calculate_all_j = ['SPREAD(t1.ts)' , 'SPREAD(t1.q_ts)' , 'SPREAD(t1.q_int)' ,'SPREAD(t1.q_bigint)' , 'SPREAD(t1.q_smallint)' ,'SPREAD(t1.q_tinyint)' ,'SPREAD(t1.q_float)' ,'SPREAD(t1.q_double)' , + 'SPREAD(t2.ts)' , 'SPREAD(t2.q_ts)' , 'SPREAD(t2.q_int)' ,'SPREAD(t2.q_bigint)' , 'SPREAD(t2.q_smallint)' ,'SPREAD(t2.q_tinyint)' ,'SPREAD(t2.q_float)' ,'SPREAD(t2.q_double)' , + '(SPREAD(t1.q_int) + SPREAD(t1.q_bigint))' , '(SPREAD(t1.q_tinyint) - SPREAD(t1.q_float))', '(SPREAD(t1.q_double) * SPREAD(t1.q_tinyint))' , '(SPREAD(t1.q_double) / SPREAD(t1.q_tinyint))', + '(SPREAD(t2.q_int) + SPREAD(t2.q_bigint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t2.q_double) * SPREAD(t2.q_tinyint))' , '(SPREAD(t2.q_double) / SPREAD(t2.q_tinyint))', + '(SPREAD(t1.q_int) + SPREAD(t1.q_smallint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t1.q_double) * SPREAD(t1.q_tinyint))' , '(SPREAD(t1.q_double) / SPREAD(t1.q_float))'] + calc_calculate_regular_j = ['DIFF(t1.q_int)' ,'DIFF(t1.q_bigint)' , 'DIFF(t1.q_smallint)' ,'DIFF(t1.q_tinyint)' ,'DIFF(t1.q_float)' ,'DIFF(t1.q_double)' , + 'DERIVATIVE(t1.q_int,15s,0)' , 'DERIVATIVE(t1.q_bigint,10s,1)' , 'DERIVATIVE(t1.q_smallint,20s,0)' ,'DERIVATIVE(t1.q_tinyint,10s,1)' ,'DERIVATIVE(t1.q_float,6s,0)' ,'DERIVATIVE(t1.q_double,3s,1)' , + 'DIFF(t2.q_int)' ,'DIFF(t2.q_bigint)' , 'DIFF(t2.q_smallint)' ,'DIFF(t2.q_tinyint)' ,'DIFF(t2.q_float)' ,'DIFF(t2.q_double)' , + 'DERIVATIVE(t2.q_int,15s,0)' , 'DERIVATIVE(t2.q_bigint,10s,1)' , 'DERIVATIVE(t2.q_smallint,20s,0)' ,'DERIVATIVE(t2.q_tinyint,10s,1)' ,'DERIVATIVE(t2.q_float,6s,0)' ,'DERIVATIVE(t2.q_double,3s,1)' ] + calc_calculate_groupbytbname_j = calc_calculate_regular_j + + + #inter && calc_aggregate_all\calc_aggregate_regular\calc_select_all + interval_sliding = ['interval(4w) sliding(1w) ','interval(1w) sliding(1d) ','interval(1d) sliding(1h) ' , + 'interval(1h) sliding(1m) ','interval(1m) sliding(1s) ','interval(1s) sliding(10a) ', + 'interval(1y) ','interval(1n) ','interval(1w) ','interval(1d) ','interval(1h) ','interval(1m) ','interval(1s) ' ,'interval(10a)', + 'interval(1y,1n) ','interval(1n,1w) ','interval(1w,1d) ','interval(1d,1h) ','interval(1h,1m) ','interval(1m,1s) ','interval(1s,10a) ' ,'interval(100a,30a)'] + + + for i in range(self.fornum): + tdSql.query("select 1-1 from table_0;") + sql = "select count(*) from (select count(*) from stable_1 where ts>= 1620000000000 interval(1d) group by tbname) interval(1d);" + tdLog.info(sql) + tdSql.query(sql) + tdSql.checkData(0,0,'2021-08-27 00:00:00.000') + tdSql.checkData(0,1,3) + tdSql.checkData(1,0,'2021-08-28 00:00:00.000') + tdSql.checkData(1,1,3) + tdSql.checkData(2,0,'2021-08-29 00:00:00.000') + tdSql.checkRows(12) + + #sql = "select * from ( select * from regular_table_1 where q_tinyint >= -127 and q_tinyint <= 127 order by ts );" + tdSql.query("select 1-2 from table_0;") + sql = "select * from ( select * from regular_table_1 where " + sql += "%s );" % random.choice(q_where) + datacheck = self.regular1_checkall_0(sql) + tdSql.checkRows(100) + datacheck = self.regular1_checkall_100(sql) + + #sql = "select * from ( select * from regular_table_1 ) where q_binary like 'binary%' or q_binary = '0' order by ts asc ;" + tdSql.query("select 1-3 from table_0;") + sql = "select * from ( select * from regular_table_1 ) where " + sql += "%s ;" % random.choice(q_where) + datacheck = self.regular1_checkall_0(sql) + tdSql.checkRows(100) + datacheck = self.regular1_checkall_100(sql) + + #sql = select * from ( select * from regular_table_1 t1 , regular_table_2 t2 where t1.ts = t2.ts and t1.q_double >= -1000000000 and t1.q_double <= 1000000000 and t2.q_double >= -1000000000 and t2.q_double <= 1000000000 order by t2.ts asc );; + tdSql.query("select 1-4 from table_0;") + sql = "select * from ( select * from regular_table_1 t1 , regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s )" % random.choice(q_u_where) + datacheck = self.regular_join_checkall_0(sql) + tdSql.checkRows(100) + datacheck = self.regular_join_checkall_100(sql) + + + + + #sql = "select * from ( select * from stable_1 where q_tinyint >= -127 and q_tinyint <= 127 order by ts );" + tdSql.query("select 2-1 from stable_1;") + sql = "select * from ( select * from stable_1 where " + sql += "%s );" % random.choice(qt_where) + datacheck = self.stable1_checkall_0(sql) + tdSql.checkRows(300) + datacheck = self.stable1_checkall_300(sql) + + #sql = "select * from ( select * from stable_1 ) order by ts asc ;" + tdSql.query("select 2-2 from stable_1;") + sql = "select * from ( select * from stable_1 ) where " + sql += "%s ;" % random.choice(qt_where) + datacheck = self.stable1_checkall_0(sql) + tdSql.checkRows(300) + datacheck = self.stable1_checkall_300(sql) + + #sql = "select * from ( select * from table_0 ) where q_binary like 'binary%' or q_binary = '0' order by ts asc ;" + tdSql.query("select 2-3 from stable_1;") + sql = "select * from ( select * from table_0 ) where " + sql += "%s ;" % random.choice(q_where) + datacheck = self.regular1_checkall_0(sql) + tdSql.checkRows(100) + datacheck = self.regular1_checkall_100(sql) + + #sql = "select * from ( select * from table_0 where q_binary like 'binary%' or q_binary = '0' order by ts asc );" + tdSql.query("select 2-4 from stable_1;") + sql = "select * from ( select * from table_0 where " + sql += "%s );" % random.choice(q_where) + datacheck = self.regular1_checkall_0(sql) + tdSql.checkRows(100) + datacheck = self.regular1_checkall_100(sql) + + #sql = select * from ( select * from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and t1.t_int = t2.t_int ) ;; + tdSql.query("select 2-5 from stable_1;") + sql = "select * from ( select * from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s )" % random.choice(t_join_where) + datacheck = self.stable_join_checkall_0(sql) + tdSql.checkRows(100) + datacheck = self.stable_join_checkall_100(sql) + + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index 7bec22b925ba41fd28a01dda84705d7263305382..81ee262ae8d6e3367d351ed98dcdbd083ec1f4fe 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -148,6 +148,22 @@ class TDTestCase: tdLog.info("case for bug_6387") self.bug_6387() + #JIRA TS-583 + tdLog.info("case for JIRA TS-583") + tdSql.execute("create database test2") + tdSql.execute("use test2") + tdSql.execute("create table stb(ts timestamp, c1 int) tags(t1 binary(120))") + tdSql.execute("create table t0 using stb tags('abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz')") + + tdSql.query("show create table t0") + tdSql.checkRows(1) + + tdSql.execute("create table stb2(ts timestamp, c1 int) tags(t1 nchar(120))") + tdSql.execute("create table t1 using stb2 tags('abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz')") + + tdSql.query("show create table t1") + tdSql.checkRows(1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryCnameDisplay.py b/tests/pytest/query/queryCnameDisplay.py index 66a7f85120fe13293996d1bd3153b6fe9b1d6a72..186b3bfe1d1d06c4210c950fff097cb37a73d5df 100644 --- a/tests/pytest/query/queryCnameDisplay.py +++ b/tests/pytest/query/queryCnameDisplay.py @@ -79,16 +79,22 @@ class TDTestCase: tdSql.execute('insert into st1 values (now, 1, 2, 1.1, 2.2, "a", 1, 1, false, "bb");') # select as cname with cname_list - sql_seq = f'select count(ts) as {cname_list[0]}, sum(pi1) as {cname_list[1]}, avg(pi2) as {cname_list[2]}, count(pf1) as {cname_list[3]}, count(pf2) as {cname_list[4]}, count(ps1) as {cname_list[5]}, min(pi3) as {cname_list[6]}, max(pi4) as {cname_list[7]}, count(pb1) as {cname_list[8]}, count(ps2) as {cname_list[9]}, count(si1) as {cname_list[10]}, count(si2) as {cname_list[11]}, count(sf1) as {cname_list[12]}, count(sf2) as {cname_list[13]}, count(ss1) as {cname_list[14]}, count(si3) as {cname_list[15]}, count(si4) as {cname_list[16]}, count(sb1) as {cname_list[17]}, count(ss2) as {cname_list[18]} from super_table_cname_check' - sql_seq_no_as = sql_seq.replace(' as ', ' ') - res = tdSql.getColNameList(sql_seq) - res_no_as = tdSql.getColNameList(sql_seq_no_as) + sql_seq1 = f'select count(ts) as {cname_list[0]}, sum(pi1) as {cname_list[1]}, avg(pi2) as {cname_list[2]}, count(pf1) as {cname_list[3]}, count(pf2) as {cname_list[4]}, count(ps1) as {cname_list[5]}, min(pi3) as {cname_list[6]}, max(pi4) as {cname_list[7]}, count(pb1) as {cname_list[8]}, count(ps2) as {cname_list[9]} from super_table_cname_check' + sql_seq2 = f'select count(si1) as {cname_list[10]}, count(si2) as {cname_list[11]}, count(sf1) as {cname_list[12]}, count(sf2) as {cname_list[13]}, count(ss1) as {cname_list[14]}, count(si3) as {cname_list[15]}, count(si4) as {cname_list[16]}, count(sb1) as {cname_list[17]}, count(ss2) as {cname_list[18]} from super_table_cname_check' + sql_seq_no_as1 = sql_seq1.replace(' as ', ' ') + sql_seq_no_as2 = sql_seq2.replace(' as ', ' ') + res1 = tdSql.getColNameList(sql_seq1) + res2 = tdSql.getColNameList(sql_seq2) + res_no_as1 = tdSql.getColNameList(sql_seq_no_as1) + res_no_as2 = tdSql.getColNameList(sql_seq_no_as2) # cname[1] > 64, it is expected to be equal to 64 cname_list_1_expected = cname_list[1][:-1] cname_list[1] = cname_list_1_expected - checkColNameList = tdSql.checkColNameList(res, cname_list) - checkColNameList = tdSql.checkColNameList(res_no_as, cname_list) + tdSql.checkColNameList(res1, cname_list[:10]) + tdSql.checkColNameList(res2, cname_list[10:]) + tdSql.checkColNameList(res_no_as1, cname_list[:10]) + tdSql.checkColNameList(res_no_as2, cname_list[10:]) def run(self): tdSql.prepare() diff --git a/tests/pytest/query/queryGroupTbname.py b/tests/pytest/query/queryGroupTbname.py index 095feed52fa7c44e2dd1b98a5f15c7034165b95e..73ee5f0fa8d2f358e34bab3336d6f64364b6af83 100644 --- a/tests/pytest/query/queryGroupTbname.py +++ b/tests/pytest/query/queryGroupTbname.py @@ -14,6 +14,7 @@ from util.log import tdLog from util.cases import tdCases from util.sql import tdSql from util.common import tdCom +import random class TDTestCase: def init(self, conn, logSql): @@ -47,6 +48,26 @@ class TDTestCase: for i in range(100): tdSql.query(f'select {table_name_sub1},{table_name_sub2},{table_name_sub3},{table_name_sub4},{table_name_sub5},{table_name_sub6},{table_name_sub7},{table_name_sub8},{table_name_sub9} from {table_name} where tbname in ("{table_name_sub1}","{table_name_sub2}","{table_name_sub3}","{table_name_sub4}","{table_name_sub5}","{table_name_sub6}","{table_name_sub7}","{table_name_sub8}","{table_name_sub9}") and ts >= "1980-01-01 00:00:00.000"') tdSql.checkRows(90) + + # TS-634 + tdLog.info("test case for bug TS-634") + tdSql.execute("create database test") + tdSql.execute("use test") + tdSql.execute("create table meters (ts TIMESTAMP,voltage INT) TAGS (tableid INT)") + tdSql.execute("CREATE TABLE t1 USING meters TAGS (1)") + tdSql.execute("CREATE TABLE t2 USING meters TAGS (2)") + + ts = 1605381041000 + for i in range(10): + tdSql.execute("INSERT INTO t1 values(%d, %d)" % (ts + i, random.randint(0, 100))) + tdSql.execute("INSERT INTO t2 values(%d, %d)" % (ts + i, random.randint(0, 100))) + + tdSql.query("select last_row(*), tbname from meters group by tbname order by ts desc") + tdSql.checkRows(2) + + tdSql.execute("INSERT INTO t2 values(now, 2)") + tdSql.query("select last_row(*), tbname from meters group by tbname order by ts desc") + tdSql.checkRows(2) def run(self): tdSql.prepare() diff --git a/tests/pytest/query/querySort.py b/tests/pytest/query/querySort.py index 17022bdc41057bcb67e1530a2cb6d399bada20ff..a50b9cbf8afd7052a78e1f6ef85b8c464e816e71 100644 --- a/tests/pytest/query/querySort.py +++ b/tests/pytest/query/querySort.py @@ -97,7 +97,7 @@ class TDTestCase: self.checkColumnSorted(0, "desc") print("======= step 2: verify order for special column =========") - + tdSql.query("select tbcol1 from st order by ts desc") tdSql.query("select tbcol6 from st order by ts desc") @@ -122,6 +122,63 @@ class TDTestCase: (i, i)) self.checkColumnSorted(1, "desc") + # order by rules: https://jira.taosdata.com:18090/pages/viewpage.action?pageId=123455481 + tdSql.error("select tbcol1 from st order by 123") + tdSql.error("select tbcol1 from st order by tbname") + tdSql.error("select tbcol1 from st order by tagcol1") + tdSql.error("select tbcol1 from st order by ''") + tdSql.error("select top(tbcol1, 12) from st1 order by tbcol1,ts") + tdSql.error("select top(tbcol1, 12) from st order by tbcol1,ts,tbcol2") + tdSql.error("select top(tbcol1, 12) from st order by ts, tbcol1") + tdSql.error("select top(tbcol1, 2) from st1 group by tbcol1 order by tbcol2") + + fun_list = ['avg','count','twa','sum','stddev','leastsquares','min', + 'max','first','last','top','bottom','percentile','apercentile', + 'last_row','diff','spread','distinct'] + key = ['tbol','tagcol'] + for i in range(1,15): + for k in key: + for j in fun_list: + if j == 'leastsquares': + pick_func=j+'('+ k + str(i) +',1,1)' + elif j == 'top' or j == 'bottom' : continue + elif j == 'percentile' or j == 'apercentile': + pick_func=j+'('+ k + str(i) +',1)' + else: + pick_func=j+'('+ k + str(i) +')' + sql = 'select %s from st group by %s order by %s' % (pick_func , k+str(i), k+str(i)) + tdSql.error(sql) + sql = 'select %s from st6 group by %s order by %s ' % (pick_func , k+str(i), k+str(i)) + tdSql.error(sql) + + tdSql.query("select top(tbcol1, 2) from st1 group by tbcol2 order by tbcol2") + tdSql.query("select top(tbcol1, 12) from st order by tbcol1, ts") + + tdSql.query("select avg(tbcol1) from st group by tbname order by tbname") + tdSql.checkData(1, 0, 5.5) + tdSql.checkData(5, 1, "st6") + + tdSql.query("select top(tbcol1, 2) from st group by tbname order by tbname") + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 2, "st2") + + tdSql.query("select top(tbcol1, 12) from st order by tbcol1") + tdSql.checkData(1, 1, 9) + + tdSql.error("select top(tbcol1, 12) from st1 order by tbcol1,ts") + tdSql.error("select top(tbcol1, 12),tbname from st order by tbcol1,tbname") + + tdSql.query("select top(tbcol1, 12) from st group by tbname order by tbname desc") + tdSql.checkData(1, 2, "st10") + tdSql.checkData(10, 2, "st9") + + tdSql.query("select top(tbcol1, 2) from st group by tbname order by tbname desc,ts") + tdSql.checkData(1, 2, "st10") + tdSql.checkData(10, 2, "st5") + tdSql.checkData(0, 0, "2018-09-17 09:00:00.109") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.110") + tdSql.checkData(2, 0, "2018-09-17 09:00:00.099") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryStateWindow.py b/tests/pytest/query/queryStateWindow.py index 251dbef65841cc17b31046320a7e966426c5eeb1..e43997eff3a8139b575537356dfb60f9692c35e3 100644 --- a/tests/pytest/query/queryStateWindow.py +++ b/tests/pytest/query/queryStateWindow.py @@ -42,8 +42,12 @@ class TDTestCase: tdSql.execute( "INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.000', 1, '2020-05-13 10:00:00.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 254, 1)('2020-05-13 10:00:01.000', 1, '2020-05-13 10:00:01.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 253, 5)('2020-05-13 10:00:02.000', 10, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', 10, -127, false, '测试', 15, 10, 65534, 253, 10)('2020-05-13 10:00:03.000', 1, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', -10, -126, true, '测试', 14, 12, 65532, 254, 15)") - for i in range(self.rowNum): - tdSql.execute("insert into dev_002 (ts,t1) values(%d, %d,)" % (self.ts + i, i + 1)) + for i in range(10): + sql = "insert into dev_002(ts, t1) values" + batch = int(self.rowNum / 10) + for j in range(batch): + sql += "(%d, %d)" % (self.ts + batch * i + j, batch * i + j) + tdSql.execute(sql) tdSql.query("select count(ts) from dev_001 state_window(t1)") tdSql.checkRows(3) @@ -101,6 +105,42 @@ class TDTestCase: tdSql.error("select count(*) from dev_001 state_window(t10)") tdSql.error("select count(*) from dev_001 state_window(tag2)") + # TS-537 + tdLog.info("case for TS-537") + tdSql.execute("create stable stb (ts timestamp, c1 int, c2 float) tags(tg1 int)") + tdSql.execute("CREATE TABLE IF NOT EXISTS db.tb1 USING db.stb TAGS (1)") + sql = "insert into tb1 values(1635398806734, 1, 1.000000)(1635398810062, 1, 2.000000)(1635398811528, 1, 3.000000)(1635398813301, 1, 4.000000)" + sql += "(1635398818507, 2, 1.000000)(1635398823464, 2, 1.000000)(1635398825150, 2, 1.000000)(1635398826453, 2, 1.000000)(1635399123037, 2, 2.000000)" + sql += "(1635399125335, 2, 2.000000)(1635399126292, 2, 2.000000)(1635399127288, 2, 2.000000)(1635399129361, 2, 2.000000)(1635399133331, 1, 2.000000)" + sql += "(1635399134179, 1, 2.000000)(1635399134909, 1, 2.000000)(1635399135617, 1, 2.000000)(1635399136372, 1, 2.000000)" + tdSql.execute(sql) + + tdSql.query("select * from (select first(ts), count(*), c1 from db.tb1 state_window(c1))") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 1, 9) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 1, 5) + tdSql.checkData(2, 2, 1) + + tdSql.query("select fts, cnt, c1 from (select first(ts) fts, count(*) cnt, c1 from db.tb1 state_window(c1))") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 1, 9) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 1, 5) + tdSql.checkData(2, 2, 1) + + tdSql.query("select * from (select first(ts) fts, count(*) cnt, c1 from db.tb1 state_window(c1))") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 1, 9) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 1, 5) + tdSql.checkData(2, 2, 1) def stop(self): tdSql.close() diff --git a/tests/pytest/query/select_last_crash.py b/tests/pytest/query/select_last_crash.py index 9b580a24acd145e2e90ae6ee513e946d72820f2c..4872f0d0f5ed94768a563e658e7af14816e272c4 100644 --- a/tests/pytest/query/select_last_crash.py +++ b/tests/pytest/query/select_last_crash.py @@ -16,7 +16,8 @@ import taos from util.log import * from util.cases import * from util.sql import * - +from util.dnodes import * +import random class TDTestCase: def init(self, conn, logSql): @@ -41,6 +42,32 @@ class TDTestCase: tdSql.query("select last(*) from st") tdSql.checkRows(1) + + # TS-717 + tdLog.info("case for TS-717") + cachelast_values = [0, 1, 3] + + for value in cachelast_values: + tdLog.info("case for cachelast value: %d" % value) + tdSql.execute("drop database if exists db") + tdLog.sleep(1) + tdSql.execute("create database db cachelast %d" % value) + tdSql.execute("use db") + tdSql.execute("create table stb(ts timestamp, c1 int, c2 binary(20), c3 binary(5)) tags(t1 int)") + + sql = "insert into t1 using stb tags(1) (ts, c1, c2) values" + for i in range(self.rowNum): + sql += "(%d, %d, 'test')" % (self.ts + i, random.randint(1,100)) + tdSql.execute(sql) + + tdSql.query("select * from stb") + tdSql.checkRows(self.rowNum) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select * from stb") + tdSql.checkRows(self.rowNum) def stop(self): tdSql.close() diff --git a/tests/pytest/query/udf.py b/tests/pytest/query/udf.py index 5b345643b30856195caab938f59c7e8f7a642784..7e04f8ad728d9197621c35fad46389804c8310ae 100644 --- a/tests/pytest/query/udf.py +++ b/tests/pytest/query/udf.py @@ -73,21 +73,18 @@ class TDTestCase: tdSql.error(sql) sql = 'select abs_max(c2) from db.stb' tdSql.query(sql) - tdSql.checkData(0,0,1410065607) + tdSql.checkData(0,0,10000000199) def test_udf_values(self): tdSql.execute("drop function abs_max") tdSql.execute("create function add_one as '/tmp/add_one.so' outputtype int") tdSql.execute("create aggregate function abs_max as '/tmp/abs_max.so' outputtype bigint;") - tdSql.execute("create aggregate function sum_double as '/tmp/sum_double.so' outputtype int bufsize 128;") + tdSql.execute("create aggregate function sum_double as '/tmp/sum_double.so' outputtype bigint;") - # UDF bug no 1 -> follow 3 cases about this bug ; # tdSql.error("create aggregate function max as '/tmp/abs_max.so' outputtype bigint ;") # tdSql.error("create aggregate function avg as '/tmp/abs_max.so' outputtype bigint ;") # tdSql.error("create aggregate function dbs as '/tmp/abs_max.so' outputtype bigint ;") - - tdSql.execute("drop database if exists test") tdSql.execute("create database test") tdSql.execute("use test") @@ -117,7 +114,7 @@ class TDTestCase: tdSql.execute("insert into bound values(%d, %d , %f, %d , %s)"%(epoch_time+1000,intdata2+1,float(intdata2+1),bigintdata2+1,"'binary"+str(intdata2+1)+"'")) # check super table calculation results - tdSql.query("select add_one(id) from st") + tdSql.query("select add_one(id) test from st") tdSql.checkData(0,0,1) tdSql.checkData(1,0,2) tdSql.checkData(4,0,5) @@ -157,29 +154,267 @@ class TDTestCase: tdLog.info(" ====== unexpected error occured about UDF function =====") sys.exit() - # UDF bug no 2 -> values of abs_max not inconsistent from common table and stable. - # tdSql.query("select abs_max(val) from st") # result is 0 rows - # tdSql.query("select abs_max(val) from tb1") - # tdSql.checkData(0,0,0) # this is error result - # tdSql.query("select sum_double(val) from st") # result is 0 rows - # tdSql.query("select sum_double(val) from tb1") - # tdSql.checkData(0,0,0) # this is error result + tdSql.query("select abs_max(val) from st") + tdSql.error("select abs_max(val),count(tbname) from st") + tdSql.query("select abs_max(val) from tb1") + tdSql.checkRows(0) + tdSql.query("select sum_double(val) from st") + tdSql.query("select sum_double(val) from tb1") + tdSql.checkRows(0) - # UDF bug no 3 -> values of abs_max will error for boundary number - # check super table calculation results - # tdSql.query("select abs_max(number) from st") - # tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select abs_max(number) from st") + tdSql.checkData(0,0,9223372036854775807) # check common table calculation results tdSql.query("select abs_max(number) from tb1") tdSql.checkData(0,0,400) tdSql.query("select abs_max(number) from tb2") tdSql.checkData(0,0,400) + tdSql.execute("select add_one(id) from st limit 10 offset 2") + tdSql.query("select add_one(id) from st where ts > 1604298064000 and ts < 1604298064020 ") + tdSql.checkData(0,0,1) + tdSql.checkData(1,0,-2147483644) + tdSql.query("select add_one(id) from tb1 where ts > 1604298064000 and ts < 1604298064020 ") + tdSql.checkData(0,0,1) + tdSql.query("select sum_double(id) from st where ts > 1604298064030 and ts < 1604298064060 ") + tdSql.checkData(0,0,14) + tdSql.query("select sum_double(id) from tb2 where ts > 1604298064030 and ts < 1604298064060 ") + tdSql.checkRows(0) + tdSql.query("select add_one(id) from st where ts = 1604298064000 ") + tdSql.checkData(0,0,-2147483645) + tdSql.query("select add_one(id) from st where ts > 1604298064000 and id in (2,3) and ind =1;") + tdSql.checkData(0,0,3) + tdSql.checkData(1,0,4) + tdSql.query("select id , add_one(id) from tb1 where ts > 1604298064000 and id in (2,3)") + tdSql.checkData(0,0,2) + tdSql.checkData(0,1,3) + tdSql.checkData(1,0,3) + tdSql.checkData(1,1,4) + tdSql.query("select sum_double(id) from tb1 where ts > 1604298064000 and id in (2,3)") + tdSql.checkData(0,0,10) + tdSql.query("select sum_double(id) from st where ts > 1604298064000 and id in (2,3) and ind =1") + tdSql.checkData(0,0,10) + tdSql.query("select abs_max(number) from st where ts > 1604298064000 and id in (2,3) and ind =1") + tdSql.checkData(0,0,300) + tdSql.query("select sum_double(id) from st where ts = 1604298064030 ") + tdSql.checkData(0,0,4) + tdSql.query("select abs_max(number) from st where ts = 1604298064100 ") + tdSql.checkData(0,0,9223372036854775806) + tdSql.query("select abs_max(number) from tb2 where ts = 1604298064100 ") + tdSql.checkData(0,0,400) + tdSql.query("select sum_double(id) from tb2 where ts = 1604298064100 ") + tdSql.checkData(0,0,8) + tdSql.query("select add_one(id) from st where ts >= 1604298064000 and ts <= 1604298064010") + tdSql.checkData(0,0,1) + tdSql.checkData(1,0,-2147483645) + tdSql.checkData(2,0,-2147483644) + tdSql.query("select add_one(id) from tb1 where ts >= 1604298064000 and ts <= 1604298064010") + tdSql.checkData(0,0,1) + tdSql.query("select sum_double(id) from st where ts >= 1604298064030 and ts <= 1604298064050") + tdSql.checkData(0,0,18) + tdSql.query("select sum_double(id) from tb2 where ts >= 1604298064030 and ts <= 1604298064100") + tdSql.checkData(0,0,20) + tdSql.query("select abs_max(number) from tb2 where ts >= 1604298064030 and ts <= 1604298064100") + tdSql.checkData(0,0,400) + tdSql.query("select abs_max(number) from st where ts >= 1604298064030 and ts <= 1604298064100") + tdSql.checkData(0,0,9223372036854775806) + tdSql.query("select id from st where id != 0 and ts >=1604298064070") + tdSql.checkData(0,0,1) + tdSql.query("select add_one(id) from st where id != 0 and ts >=1604298064070") + tdSql.checkData(0,0,2) + tdSql.query("select add_one(id) from st where id <> 0 and ts >=1604298064010") + tdSql.checkData(0,0,2) + tdSql.query("select sum_double(id) from st where id in (2,3,4) and ts >=1604298064070") + tdSql.checkData(0,0,18) + tdSql.query("select sum_double(id) from tb2 where id in (2,3,4) and ts >=1604298064070") + tdSql.checkData(0,0,18) + tdSql.query("select abs_max(number) from st where id in (2,3,4) and ts >=1604298064070") + tdSql.checkData(0,0,400) + tdSql.query("select add_one(id) from st where id = 0 ") + tdSql.checkData(0,0,1) + tdSql.checkData(1,0,1) + tdSql.query("select add_one(id) from tb2 where id = 0 ") + tdSql.checkData(0,0,1) + tdSql.query("select sum_double(id) from st where id = 1") + tdSql.checkData(0,0,4) + tdSql.query("select sum_double(id) from tb2 where id = 1") + tdSql.checkData(0,0,2) + + + tdSql.query("select add_one(id) from st where id is not null and ts >=1604298065000 ") + tdSql.checkData(0,0,None) + tdSql.query("select abs_max(number) from st where id is not null and ts >=1604298065000 ") + tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select abs_max(number) from bound where id is not null and ts >=1604298065000 ") + tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select sum_double(id) from st where id is not null and ts >=1604298064000 and ind = 1 ") + tdSql.checkData(0,0,20) + tdSql.query("select sum_double(id) from tb1 where id is not null and ts >=1604298064000 ") + tdSql.checkData(0,0,20) + tdSql.query("select add_one(id) from st where id is null and ts >=1604298065000 ") + tdSql.checkRows(0) + tdSql.query("select abs_max(number) from st where id is null and ts >=1604298065000 ") + tdSql.checkRows(0) + tdSql.query("select abs_max(number) from tb1 where id is null and ts >=1604298065000 ") + tdSql.checkRows(0) + tdSql.query("select add_one(id) from bound where id is not null and ts >=1604298065000;") + tdSql.checkData(0,0,None) + tdSql.query("select id,add_one(id) from bound;") + tdSql.checkRowCol(4,2) + tdSql.checkData(3,1,None) + tdSql.query("select add_one(id) from st where ts between 1604298064000 and 1604298064010") + tdSql.checkRows(3) + tdSql.query("select add_one(id) from tb1 where ts between 1604298064000 and 1604298064010") + tdSql.checkRows(1) + tdSql.query("select sum_double(id) from st where ts between 1604298064000 and 1604298064010 and id>=0") + tdSql.checkData(0,0,0) + tdSql.query("select sum_double(id) from tb1 where ts between 1604298064000 and 1604298064010 and id>=0") + tdSql.checkData(0,0,0) + tdSql.query("select add_one(id) from st where id in (1,2)") + tdSql.checkData(0,0,2) + tdSql.checkData(1,0,3) + tdSql.checkData(2,0,2) + tdSql.checkData(3,0,3) + tdSql.checkRows(4) + + tdSql.query("select sum_double(id) from st where ts < now and ind =1 interval(1s)") + tdSql.checkData(0,1,20) + tdSql.error("select sum_double(id) from st where ts < now and ind =1 interval(3s) sliding (1s) fill (NULL) ") + tdSql.error("select sum_double(id) from st session(ts, 1s)") + tdSql.query("select sum_double(id) from tb1 session(ts, 1s)") + tdSql.checkData(0,1,20) + + # intervals sliding values calculation + tdSql.query("select sum_double(id) from st where ts < now and ind =1 interval(3s) sliding (1s) limit 2") + tdSql.checkData(0,1,20) + tdSql.checkData(1,1,20) + + # scalar_function can't work when using interval and sliding ========= + tdSql.error("select add_one(id) from st where ts < now and ind =1 interval(3s) sliding (1s) limit 2 ") + tdSql.error("select add_one(id) from st order by ts") + tdSql.error("select ts,id,add_one(id) from st order by ts asc;") + + # # UDF not support order by + tdSql.error("select ts,id,add_one(id) from st order by ts desc;") + + # UDF function union all + tdSql.query("select add_one(id) from tb1 union all select add_one(id) from tb2;") + tdSql.checkRows(10) + tdSql.checkData(0,0,1) + tdSql.checkData(5,0,1) + tdSql.query("select sum_double(id) from tb1 union all select sum_double(id) from tb2;") + tdSql.checkRows(2) + tdSql.checkData(0,0,20) + tdSql.checkData(1,0,20) + tdSql.query("select abs_max(number) from tb1 union all select abs_max(number) from bound;") + tdSql.checkRows(2) + tdSql.checkData(0,0,400) + tdSql.checkData(1,0,9223372036854775807) + tdSql.execute("create stable stb (ts timestamp,id int , val double , number bigint, chars binary(200)) tags (ind int)") + tdSql.execute("create table stb1 using stb tags(3)") + tdSql.execute("insert into stb1 values(1604298064000 , 1 , 1.0 , 10000 ,'chars')") + tdSql.query("select add_one(id) from st union all select add_one(id) from stb;") + tdSql.checkRows(15) + tdSql.checkData(13,0,None) + tdSql.checkData(14,0,2) + tdSql.query("select add_one(id) from st union all select add_one(id) from stb1;") + tdSql.checkRows(15) + tdSql.checkData(13,0,None) + tdSql.checkData(14,0,2) + tdSql.query("select id ,add_one(id) from tb1 union all select id ,add_one(id) from stb1;") + tdSql.checkRows(6) + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,2) + + # aggregate union all for different stables + tdSql.query("select sum_double(id) from st union all select sum_double(id) from stb;") + tdSql.checkRows(2) + tdSql.checkData(0,0,44) + tdSql.checkData(1,0,2) + tdSql.query("select id from st union all select id from stb1;") + tdSql.checkRows(15) + tdSql.query("select id from tb1 union all select id from stb1") + tdSql.checkRows(6) + tdSql.query("select sum_double(id) from tb1 union all select sum_double(id) from stb") + tdSql.checkData(0,0,20) + tdSql.checkData(1,0,2) + tdSql.query("select sum_double(id) from st union all select sum_double(id) from stb1;") + tdSql.checkRows(2) + tdSql.checkData(0,0,44) + tdSql.checkData(1,0,2) + tdSql.query("select abs_max(number) from st union all select abs_max(number) from stb;") + tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select abs_max(number) from bound union all select abs_max(number) from stb1;") + tdSql.checkData(0,0,9223372036854775807) + tdSql.checkData(1,0,10000) + tdSql.query("select abs_max(number) from st union all select abs_max(number) from stb1;") + tdSql.checkData(0,0,9223372036854775807) + tdSql.checkData(1,0,10000) + + # group by for aggegate function ; + tdSql.query("select sum_double(id) from st group by tbname;") + tdSql.checkData(0,0,20) + tdSql.checkData(0,1,'tb1') + tdSql.checkData(1,0,20) + tdSql.checkData(1,1,'tb2') + tdSql.query("select sum_double(id) from st group by id;") + tdSql.checkRows(9) + tdSql.query("select sum_double(id) from st group by ts") + tdSql.checkRows(12) + tdSql.query("select sum_double(id) from st group by ind") + tdSql.checkRows(3) + tdSql.query("select sum_double(id) from st group by tbname order by ts asc;") + tdSql.query("select abs_max(number) from st group by id") + tdSql.checkRows(9) + tdSql.checkData(0,0,9223372036854775806) + tdSql.checkData(8,0,9223372036854775807) + tdSql.query("select abs_max(number) from st group by ts") + tdSql.checkRows(12) + tdSql.checkData(11,0,9223372036854775807) + tdSql.checkData(1,0,9223372036854775805) + tdSql.query("select abs_max(number) from st group by ind") + tdSql.checkRows(3) + tdSql.checkData(0,0,400) + tdSql.checkData(2,0,9223372036854775807) + + # UDF join + tdSql.query("select add_one(tb1.id),add_one(bound.id) from tb1,bound where tb1.ts=bound.ts;") + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,-2147483644) + tdSql.query("select stb1.ts,add_one(stb1.id),bound.ts,add_one(bound.id) from stb1,bound where stb1.ts=bound.ts") + tdSql.checkData(0,1,2) + tdSql.checkData(0,3,-2147483645) + tdSql.query("select st.ts,add_one(st.id),stb.ts,add_one(stb.id) from st,stb where st.ts=stb.ts and st.ind=stb.ind") + tdSql.checkData(0,1,-2147483645) + tdSql.checkData(0,3,2) + + tdSql.query("select sum_double(tb1.id),sum_double(bound.id) from tb1,bound where tb1.ts=bound.ts;") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,-4294967290) + tdSql.query("select sum_double(stb1.id),sum_double(bound.id) from stb1,bound where stb1.ts=bound.ts") + tdSql.checkData(0,0,2) + tdSql.checkData(0,1,-4294967292) + + #UDF join for stables + tdSql.query("select sum_double(st.id),sum_double(stb.id) from st,stb where st.ts=stb.ts and st.ind=stb.ind") + tdSql.checkData(0,0,-4294967292) + tdSql.checkData(0,1,2) + tdSql.query("select abs_max(tb1.number),abs_max(bound.number) from tb1,bound where tb1.ts=bound.ts;") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,9223372036854775805) + tdSql.query("select abs_max(stb1.number),abs_max(bound.number) from stb1,bound where stb1.ts=bound.ts") + tdSql.checkData(0,0,10000) + tdSql.checkData(0,1,9223372036854775806) + tdSql.query("select abs_max(st.number),abs_max(stb.number) from st,stb where st.ts=stb.ts and st.ind=stb.ind") + tdSql.checkData(0,0,9223372036854775806) + tdSql.checkData(0,1,10000) # check boundary - # tdSql.query("select abs_max(number) from bound") - # tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select abs_max(number) from bound") + tdSql.checkData(0,0,9223372036854775807) tdLog.info("======= UDF function sum_double check =======") @@ -189,14 +424,10 @@ class TDTestCase: tdSql.query("select sum_double(id) from tb1") tdSql.checkData(0,0,20) - # UDF bug no 4 -> values error while two function work : it is limit that udf can't work with build-in functions. - # tdSql.query("select sum_double(id) , abs_max(number) from tb1") - # tdSql.checkData(0,0,20) - # tdSql.checkData(0,0,400) - - # tdSql.query("select sum_double(id) , abs_max(number) from st") - # tdSql.checkData(0,0,44) - # tdSql.checkData(0,0,9223372036854775807) + # only one udf function in SQL can use ,follow errors notice. + tdSql.error("select sum_double(id) , abs_max(number) from tb1") + tdSql.error("select sum_double(id) , abs_max(number) from st") + # UDF not support mix up with build-in functions # it seems like not support scalar_function mix up with aggregate functions @@ -204,147 +435,162 @@ class TDTestCase: tdSql.error("select sum_double(id) ,add_one(id) from tb1") tdSql.error("select sum_double(id) ,max(id) from st") tdSql.error("select sum_double(id) ,max(id) from tb1") + tdSql.error("select twa(id),add_one(id) from st") + tdSql.error("select twa(id),add_one(id) from tb1") # UDF function not support Arithmetic =================== + tdSql.query("select max(id) + 5 from st") tdSql.query("select max(id) + 5 from tb1") tdSql.query("select max(id) + avg(val) from st") + tdSql.query("select abs_max(number)*5 from st") + tdSql.checkData(0,0,46116860184273879040.000000000) + tdSql.query("select abs_max(number)*5 from tb1") + tdSql.checkData(0,0,2000.000000000) tdSql.query("select max(id) + avg(val) from tb1") - tdSql.error("select abs_max(number) + 5 from st") - tdSql.error("select abs_max(number) + 5 from tb1") + tdSql.query("select add_one(id) + 5 from st") + tdSql.checkData(4,0,10.000000000) + tdSql.query("select add_one(id)/5 from tb1") + tdSql.checkData(4,0,1.000000000) + tdSql.query("select sum_double(id)-5 from st") + tdSql.checkData(0,0,39.000000000) + tdSql.query("select sum_double(id)*5 from tb1") + tdSql.checkData(0,0,100.000000000) + + + tdSql.query("select abs_max(number) + 5 from tb1") tdSql.error("select abs_max(number) + max(id) from st") - tdSql.error("select abs_max(number)*abs_max(val) from st") + tdSql.query("select abs_max(number)*abs_max(val) from st") + tdSql.query("select sum_double(id) + sum_double(id) from st") + tdSql.checkData(0,0,88.000000000) tdLog.info("======= UDF Nested query test =======") tdSql.query("select sum(id) from (select id from st)") tdSql.checkData(0,0,22) - #UDF bug no 5 -> not support Nested query - # tdSql.query("select abs_max(number) from (select number from st)") - # tdSql.checkData(0,0,9223372036854775807) - # tdSql.query("select abs_max(number) from (select number from bound)") - # tdSql.checkData(0,0,9223372036854775807) - # tdSql.query("select sum_double(id) from (select id from st)") - # tdSql.checkData(0,0,44) - # tdSql.query("select sum_double(id) from (select id from tb1)") - # tdSql.checkData(0,0,10) + #UDF bug -> Nested query + # outer nest query + tdSql.query("select abs_max(number) from (select number from st)") + tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select abs_max(number) from (select number from bound)") + tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select sum_double(id) from (select id from st)") + tdSql.checkData(0,0,44) + tdSql.query("select sum_double(id) from (select id from bound)") + tdSql.checkData(0,0,4) + tdSql.query("select add_one(id) from (select id from st);") + tdSql.checkRows(14) + tdSql.checkData(1,0,2) + tdSql.query("select add_one(id) from (select id from bound);") + tdSql.checkRows(4) + tdSql.checkData(1,0,-2147483644) - # UDF bug no 6 -> group by work error - tdLog.info("======= UDF work with group by =======") + # inner nest query + tdSql.query("select id from (select add_one(id) id from st)") + tdSql.checkRows(14) + tdSql.checkData(13,0,None) + tdSql.query("select id from (select add_one(id) id from bound)") + tdSql.checkRows(4) + tdSql.checkData(3,0,None) - # tdSql.query("select sum_double(id) from st group by tbname;") - # tdSql.checkData(0,0,6) - # tdSql.checkData(0,1,'tb1') - # tdSql.checkData(1,0,2) - # tdSql.checkData(1,1,'tb2') - # tdSql.query("select sum_double(id) from st group by id;") - # tdSql.checkRows(2) - # tdSql.query("select sum_double(id) from st group by tbname order by ts asc;") + tdSql.query("select id from (select sum_double(id) id from bound)") + tdSql.checkData(0,0,4) + tdSql.query("select id from (select sum_double(id) id from st)") # it will crash taos shell + tdSql.checkData(0,0,44) + tdSql.query("select id from (select abs_max(number) id from st)") # it will crash taos shell + tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select id from (select abs_max(number) id from bound)") + tdSql.checkData(0,0,9223372036854775807) - tdSql.query("select sum_double(id) from st where ts < now and ind =1 interval(1s)") - tdSql.checkData(0,1,20) - tdSql.error("select sum_double(id) from st session(ts, 1s) interval (10s,1s) sliding(10s) fill (NULL) ") - tdSql.error("select sum_double(id) from st session(ts, 1s)") - tdSql.query("select sum_double(id) from tb1 session(ts, 1s)") - tdSql.checkData(0,1,20) + # inner and outer nest query - # UDF -> bug no 7 : intervals sliding values calculation error - # tdSql.query("select sum_double(id) from st where ts < now and ind =1 interval(3s) sliding (1s) limit 2") - # tdSql.checkData(0,1,20) - # tdSql.checkData(1,1,20) + tdSql.query("select add_one(id) from (select add_one(id) id from st)") + tdSql.checkRows(14) + tdSql.checkData(0,0,2) + tdSql.checkData(1,0,3) + + tdSql.query("select add_one(id) from (select add_one(id) id from tb1)") + tdSql.checkRows(5) + tdSql.checkData(0,0,2) + tdSql.checkData(1,0,3) + + tdSql.query("select sum_double(sumdb) from (select sum_double(id) sumdb from st)") + tdSql.query("select sum_double(sumdb) from (select sum_double(id) sumdb from tb1)") - # scalar_function can't work when using interval and sliding ========= - tdSql.error("select add_one(id) from st where ts < now and ind =1 interval(3s) sliding (1s) limit 2 ") + tdSql.query("select abs_max(number) from (select abs_max(number) number from st)") + tdSql.checkData(0,0,9223372036854775807) + + tdSql.query("select abs_max(number) from (select abs_max(number) number from bound)") + tdSql.checkData(0,0,9223372036854775807) + + # nest inner and outer with build-in func + + tdSql.query("select max(number) from (select abs_max(number) number from st)") + tdSql.checkData(0,0,9223372036854775807) + + tdSql.query("select max(number) from (select abs_max(number) number from bound)") + tdSql.checkData(0,0,9223372036854775807) + + tdSql.query("select sum_double(sumdb) from (select sum_double(id) sumdb from st)") + + tdSql.query("select sum(sumdb) from (select sum_double(id) sumdb from tb1)") + tdSql.checkData(0,0,20) + tdLog.info(" =====================test illegal creation method =====================") - tdSql.execute("drop function add_one") + # tdSql.execute("drop function add_one") tdSql.execute("drop function abs_max") tdSql.execute("drop function sum_double") tdSql.execute("create aggregate function error_use1 as '/tmp/abs_max.so' outputtype bigint ") tdSql.error("select error_use1(number) from st") - # UDF -> bug no 8: error return values when create aggregate functions as an scalar_function + # illega UDF create aggregate functions as an scalar_function # with no aggregate - # tdSql.execute("create function abs_max as '/tmp/abs_max.so' outputtype bigint bufsize 128") - # tdSql.query("select abs_max(number) from st") # this bug will return 3 rows - # tdSql.checkRows(1) - # tdSql.execute("create function sum_double as '/tmp/sum_double.so' outputtype bigint bufsize 128") - # tdSql.execute("select sum_double(id) from st") - # tdSql.checkRows(1) - - # UDF -> bug no 9: give bufsize for scalar_function add_one; - # UDF -> need improve : when outputtype is not match datatype which is defined in function codes - tdSql.execute("create function add_one as '/tmp/add_one.so' outputtype bigint bufsize 128") - # tdSql.error("select add_one(val) from st") # it should return error not [] for not match col datatype - # tdSql.query("select add_one(id) from st") # return error query result - # tdSql.checkData(0,0,1) - # tdSql.checkData(1,0,2) - # tdSql.checkData(5,0,1) - # tdSql.checkData(10,0,-2147483645) - # tdSql.checkData(13,0,None) + tdSql.execute("create function abs_max as '/tmp/abs_max.so' outputtype bigint bufsize 128") + tdSql.error("select abs_max(number) from st") + tdSql.execute("create function sum_double as '/tmp/sum_double.so' outputtype bigint bufsize 128") + tdSql.error("select sum_double(id) from st") # UDF -> improve : aggregate function with no bufsize : it seems with no affect - # tdSql.execute("drop function abs_max") - # tdSql.execute("drop function sum_double") + tdSql.execute("drop function abs_max") + tdSql.execute("drop function sum_double") tdSql.execute("create aggregate function abs_max as '/tmp/abs_max.so' outputtype bigint ") tdSql.execute("create aggregate function sum_double as '/tmp/sum_double.so' outputtype int ") tdSql.query("select sum_double(id) from st") tdSql.checkData(0,0,44) tdSql.query("select sum_double(id) from tb1") tdSql.checkData(0,0,20) - # tdSql.query("select abs_max(number) from st") - # tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select abs_max(number) from st") + tdSql.checkData(0,0,9223372036854775807) tdSql.query("select abs_max(number) from tb1") tdSql.checkData(0,0,400) - - #UDF bug no 10 -> create function datatype of outputtype not match col datatype - tdSql.execute("drop function abs_max") - tdSql.execute("drop function sum_double") - tdSql.execute("drop function add_one") - tdSql.execute("create function add_one as '/tmp/add_one.so' outputtype bigint;") - tdSql.execute("create aggregate function abs_max as '/tmp/abs_max.so' outputtype int bufsize 128;") - tdSql.execute("create aggregate function sum_double as '/tmp/sum_double.so' outputtype double bufsize 128;") - # tdSql.query("select sum_double(id) from st") this bug will return 0.000000 - # tdSql.checkData(0,0,44) - # tdSql.query("select sum_double(id) from tb1") - # tdSql.checkData(0,0,20) this bug will return 0.000000 - # tdSql.query("select add_one(id) from st") this bug will return series error values - # tdSql.checkData(0,0,1) - # tdSql.checkData(1,0,2) - # tdSql.checkData(5,0,1) - # tdSql.checkData(10,0,-2147483645) - # tdSql.checkData(13,0,None) - # tdSql.query("select add_one(id) from tb1") this bug will return series error values - # tdSql.checkData(0,0,1) - # tdSql.checkData(2,0,3) - # tdSql.query("select abs_max(id) from st") - # tdSql.checkData(0,0,9223372036854775807) + tdSql.query("select abs_max(number) from tb1") # it seems work well tdSql.checkData(0,0,400) + # UDF scalar function not support group by + tdSql.error("select add_one(id) from st group by tbname") - # UDF bug no 11 -> follow test case will coredump for taosd and let data lost - # tdSql.query("select add_one(id) from st group by tbname") - - # UDF -> bug no 12: give aggregate for scalar_function add_one ,it will let taosd coredump as data lost - # tdSql.execute("drop function add_one") - # tdSql.execute("create aggregate function add_one as '/tmp/add_one.so' outputtype bigint bufsize 128") - # tdSql.query("select add_one(id) from st") - - # UDF bug no 13 -> follow test case will coredump for taosc - # tdSql.query("select add_one(*) from st ") - # tdSql.query("select add_one(*) from tb1 ") - - # UDF bug no 14 -> follow test case will coredump for taosc - # tdSql.query("select abs_max(id),abs_max(number) from st ") - # tdSql.query("select abs_max(number),abs_max(number) from st ") - # tdSql.query("select sum_double(id),sum_double(id) from st ") + # UDF : give aggregate for scalar_function add_one ,it can't work well + tdSql.execute("drop function add_one") + tdSql.execute("create aggregate function add_one as '/tmp/add_one.so' outputtype bigint bufsize 128") + tdSql.error("select add_one(id) from st") + + # udf must give col list + tdSql.error("select add_one(*) from st ") + tdSql.error("select add_one(*) from tb1 ") + + # one udf function can multi use + tdSql.query("select abs_max(id),abs_max(number) from st ") + tdSql.query("select abs_max(number),abs_max(number)*3 from st ") + tdSql.query("select abs_max(number),abs_max(number)*3 from tb1 ") + tdSql.query("select sum_double(id),sum_double(id) from st ") def run(self): tdSql.prepare() @@ -366,4 +612,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/unionAllTest.py b/tests/pytest/query/unionAllTest.py index 3064e2f63e871e5c90d03d19bf125447714dd6cb..eaef5d2d573ba3c0ab08ed7ede991d25913c2454 100644 --- a/tests/pytest/query/unionAllTest.py +++ b/tests/pytest/query/unionAllTest.py @@ -24,7 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.ts = 1500000000000 + self.ts = 1600000000000 self.num = 10 def run(self): @@ -84,10 +84,77 @@ class TDTestCase: tdSql.query("select 'dcs' as options from stb where col > 200 limit 1 union all select 'aaa' as options from stb limit 10") tdSql.checkData(0, 0, 'aaa') + # https://jira.taosdata.com:18080/browse/TS-444 + tdLog.info("test case for TS-444") + + tdSql.query("select count(*) as count, loc from st where ts between 1600000000000 and 1600000000010 group by loc") + tdSql.checkRows(6) + + tdSql.query("select count(*) as count, loc from st where ts between 1600000000020 and 1600000000030 group by loc") + tdSql.checkRows(0) + + tdSql.query(''' select count(*) as count, loc from st where ts between 1600000000000 and 1600000000010 group by loc + union all + select count(*) as count, loc from st where ts between 1600000000020 and 1600000000030 group by loc''') + tdSql.checkRows(6) + + tdSql.query(''' select count(*) as count, loc from st where ts between 1600000000020 and 1600000000030 group by loc + union all + select count(*) as count, loc from st where ts between 1600000000000 and 1600000000010 group by loc''') + tdSql.checkRows(6) + + # https://jira.taosdata.com:18080/browse/TS-715 + tdLog.info("test case for TS-715") + sql = "" + + tdSql.execute("create table st2(ts timestamp, c1 int, c2 int, c3 int) tags(loc nchar(20))") + + for i in range(101): + if i == 0: + sql = "select last(*) from sub0 " + else: + sql += f"union all select last(*) from sub{i} " + + tdSql.execute("create table sub%d using st2 tags('nchar%d')" % (i, i)) + tdSql.execute("insert into sub%d values(%d, %d, %d, %d)(%d, %d, %d, %d)" % (i, self.ts + i, i, i, i,self.ts + i + 101, i + 101, i + 101, i + 101)) + + tdSql.error(sql) + + # TS-795 + tdLog.info("test case for TS-795") + + functions = ["*", "count", "avg", "twa", "irate", "sum", "stddev", "leastsquares", "min", "max", "first", "last", "top", "bottom", "percentile", "apercentile", "last_row"] + + for func in functions: + expr = func + if func == "top" or func == "bottom": + expr += "(c1, 1)" + elif func == "percentile" or func == "apercentile": + expr += "(c1, 0.5)" + elif func == "leastsquares": + expr = func + "(c1, 1, 1)" + elif func == "*": + expr = func + else: + expr += "(c1)" + + for i in range(100): + if i == 0: + sql = f"select {expr} from sub0 " + else: + sql += f"union all select {expr} from sub{i} " + + tdSql.query(sql) + if func == "*": + tdSql.checkRows(200) + else: + tdSql.checkRows(100) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/create.py b/tests/pytest/table/create.py index ecd4d011418ed7eca0d028262159c5614c0f490c..51277f28e05a6b9c2a9ce0b7375e6f3b9dd98b76 100644 --- a/tests/pytest/table/create.py +++ b/tests/pytest/table/create.py @@ -28,6 +28,8 @@ class TDTestCase: print("==============step1") print("prepare data") + tdSql.error("create table db.st1 (ts timestamp(8), i int) tags(j int)") + tdSql.error("create table db.st2 (ts timestamp, i int(2)) tags(j int)") tdSql.execute("create table db.st (ts timestamp, i int) tags(j int)") tdSql.execute("create table db.tb using st tags(1)") tdSql.execute("insert into db.tb values(now, 1)") @@ -61,6 +63,9 @@ class TDTestCase: tdSql.execute("use db2") tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") tdSql.error("insert into db2.tb6 using db2.stb tags(1) values(now 1) tb2 using db2. tags( )values(now 2)") + + # case for TD-10691 + tdSql.error("create table ttb1(ts timestamp, file int )") def stop(self): tdSql.close() diff --git a/tests/pytest/table/create_db_from_normal_db.py b/tests/pytest/table/create_db_from_normal_db.py new file mode 100644 index 0000000000000000000000000000000000000000..8b5182c3b16ca31b2bbf966df294e2c4e4c12ff3 --- /dev/null +++ b/tests/pytest/table/create_db_from_normal_db.py @@ -0,0 +1,45 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("test case for TS-783") + tdSql.execute("drop table if exists db.state1;") + tdSql.execute("create table db.state1 (ts timestamp, c1 int);") + tdSql.error("create table db.test1 using db.state1 tags('tt');") + + tdSql.execute("drop table if exists db.state2;") + tdSql.execute("create table db.state2 (ts timestamp, c1 int) tags (t binary(20));") + tdSql.query("create table db.test2 using db.state2 tags('tt');") + tdSql.error("create table db.test22 using db.test2 tags('tt');") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/alter_tag.py b/tests/pytest/tag_lite/alter_tag.py index 9e5abb6c134840ecb4ab52c7d3a6ab623885e12b..c4d738e4cf0d9bd63f7813e9d267080f9f045fac 100644 --- a/tests/pytest/tag_lite/alter_tag.py +++ b/tests/pytest/tag_lite/alter_tag.py @@ -30,7 +30,7 @@ class TDTestCase: print("==============step1") tdSql.execute( - "CREATE TABLE IF NOT EXISTS ampere (ts TIMESTAMP(8),ampere DOUBLE(8)) TAGS (device_name BINARY(50),build_id BINARY(50),project_id BINARY(50),alias BINARY(50))") + "CREATE TABLE IF NOT EXISTS ampere (ts TIMESTAMP,ampere DOUBLE) TAGS (device_name BINARY(50),build_id BINARY(50),project_id BINARY(50),alias BINARY(50))") tdSql.execute("insert into d1001 using ampere tags('test', '2', '2', '2') VALUES (now, 123)") tdSql.execute("ALTER TABLE ampere ADD TAG variable_id BINARY(50)") diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py index 95c1a731bcde4c78d0aa272183bd2e7b7a4b168b..cd5a0b4c413d11b3030042f41faf2e1c6f3a4002 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanosubscribe.py @@ -59,6 +59,7 @@ class TDTestCase: assert subResult == expectResult , "Queryfile:%s ,result is %s != expect: %s" % args0 def run(self): + tdSql.prepare() buildPath = self.getBuildPath() if (buildPath == ""): tdLog.exit("taosd not found!") @@ -66,24 +67,38 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - # clear env - os.system("ps -ef |grep 'taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json' |grep -v 'grep' |awk '{print $2}'|xargs kill -9") + # clear envs + + os.system("ps -aux |grep 'taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json' |awk '{print $2}'|xargs kill -9 >/dev/null 2>&1") + os.system("ps -aux |grep 'tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json' |awk '{print $2}'|xargs kill -9 >/dev/null 2>&1") os.system("rm -rf ./subscribe_res*") os.system("rm -rf ./all_subscribe_res*") - # insert data - os.system("%staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json" % binPath) - os.system("nohup %staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json &" % binPath) - query_pid = int(subprocess.getstatusoutput('ps aux|grep "taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json" |grep -v "grep"|awk \'{print $2}\'')[1]) - + os.system("nohup %staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json & >/dev/null 2>&1" % binPath) + sleep(5) + tdSql.query("select count(*) from subnsdb.stb0") + if tdSql.checkData(0,0,100): + pass + else: + sleep(5) + tdSql.query("select count(*) from subnsdb.stb0") # if records not write done ,sleep and wait records write done! + + os.system(" nohup %staosdemo -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json & >/dev/null 2>&1" % binPath) + sleep(5) + if os.path.exists("./subscribe_res0.txt") and os.path.exists("./subscribe_res1.txt") and os.path.exists("./subscribe_res2.txt"): + pass + else: + sleep(5) # make sure query is ok + print('taosdemo query done!') + # merge result files - sleep(5) + os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") os.system("cat subscribe_res1.txt* > all_subscribe_res1.txt") os.system("cat subscribe_res2.txt* > all_subscribe_res2.txt") - + sleep(5) # correct subscribeTimes testcase subTimes0 = self.subTimes("all_subscribe_res0.txt") @@ -103,19 +118,18 @@ class TDTestCase: os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt") subTimes0 = self.subTimes("all_subscribe_res0.txt") - print("pass") self.assertCheck("all_subscribe_res0.txt",subTimes0 ,202) - - - # correct data testcase - os.system("kill -9 %d" % query_pid) sleep(3) os.system("rm -rf ./subscribe_res*") os.system("rm -rf ./all_subscribe*") os.system("rm -rf ./*.py.sql") + os.system("rm -rf ./nohup*") + os.system("ps -aux |grep 'taosdemoAllTest/taosdemoTestSupportNanoSubscribe.json' |awk '{print $2}'|xargs kill -9 >/dev/null 2>&1") + os.system("ps -aux |grep 'tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoSubscribe.json' |awk '{print $2}'|xargs kill -9 >/dev/null 2>&1") + os.system("ps -aux |grep 'tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json' |awk '{print $2}'|xargs kill -9 >/dev/null 2>&1") + - def stop(self): tdSql.close() @@ -123,3 +137,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py index 6021c9136ad235f3e9d07bb4f6654fdac54989e5..fa25629bb3633a1c5280e20662ee2c13c7ee76e0 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py @@ -20,14 +20,16 @@ from util.dnodes import * import time from datetime import datetime import ast +import re # from assertpy import assert_that import subprocess + class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -40,52 +42,54 @@ class TDTestCase: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + buildPath = root[:len(root) - len("/build/bin")] break return buildPath # 获取taosc接口查询的结果文件中的内容,返回每行数据,并断言数据的第一列内容。 - def assertfileDataTaosc(self,filename,expectResult): + def assertfileDataTaosc(self, filename, expectResult): self.filename = filename self.expectResult = expectResult - with open("%s" % filename, 'r+') as f1: + with open("%s" % filename, 'r+') as f1: for line in f1.readlines(): queryResult = line.strip().split()[0] - self.assertCheck(filename,queryResult,expectResult) + self.assertCheck(filename, queryResult, expectResult) # 获取restful接口查询的结果文件中的关键内容,目前的关键内容找到第一个key就跳出循，所以就只有一个数据。后续再修改多个结果文件。 - def getfileDataRestful(self,filename): + def getfileDataRestful(self, filename): self.filename = filename - with open("%s" % filename, 'r+') as f1: + with open("%s" % filename, 'r+') as f1: for line in f1.readlines(): contents = line.strip() if contents.find("data") != -1: + pattern = re.compile("{.*}") + contents = pattern.search(contents).group() contentsDict = ast.literal_eval(contents) # 字符串转换为字典 queryResult = contentsDict['data'][0][0] break return queryResult # 获取taosc接口查询次数 - def queryTimesTaosc(self,filename): + def queryTimesTaosc(self, filename): self.filename = filename - command = 'cat %s |wc -l'% filename - times = int(subprocess.getstatusoutput(command)[1]) + command = 'cat %s |wc -l' % filename + times = int(subprocess.getstatusoutput(command)[1]) return times # 获取restful接口查询次数 - def queryTimesRestful(self,filename): + def queryTimesRestful(self, filename): self.filename = filename - command = 'cat %s |grep "200 OK" |wc -l'% filename - times = int(subprocess.getstatusoutput(command)[1]) + command = 'cat %s |grep "200 OK" |wc -l' % filename + times = int(subprocess.getstatusoutput(command)[1]) return times # 定义断言结果是否正确。不正确返回错误结果，正确即通过。 - def assertCheck(self,filename,queryResult,expectResult): + def assertCheck(self, filename, queryResult, expectResult): self.filename = filename self.queryResult = queryResult self.expectResult = expectResult args0 = (filename, queryResult, expectResult) - assert queryResult == expectResult , "Queryfile:%s ,result is %s != expect: %s" % args0 + assert queryResult == expectResult, "Queryfile:%s ,result is %s != expect: %s" % args0 def run(self): buildPath = self.getBuildPath() @@ -93,109 +97,144 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - + binPath = buildPath + "/build/bin/" + # delete useless files - os.system("rm -rf ./query_res*") + os.system("rm -rf ./query_res*") os.system("rm -rf ./all_query*") - - # taosc query: query specified table and query super table - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/queryTaosc.json" % binPath) + + # taosc query: query specified table and query super table + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryTaosc.json" % + binPath) os.system("cat query_res0.txt* > all_query_res0_taosc.txt") os.system("cat query_res1.txt* > all_query_res1_taosc.txt") os.system("cat query_res2.txt* > all_query_res2_taosc.txt") - - # correct Times testcases + + # correct Times testcases queryTimes0Taosc = self.queryTimesTaosc("all_query_res0_taosc.txt") - self.assertCheck("all_query_res0_taosc.txt",queryTimes0Taosc,6) + self.assertCheck("all_query_res0_taosc.txt", queryTimes0Taosc, 6) queryTimes1Taosc = self.queryTimesTaosc("all_query_res1_taosc.txt") - self.assertCheck("all_query_res1_taosc.txt",queryTimes1Taosc,6) + self.assertCheck("all_query_res1_taosc.txt", queryTimes1Taosc, 6) queryTimes2Taosc = self.queryTimesTaosc("all_query_res2_taosc.txt") - self.assertCheck("all_query_res2_taosc.txt",queryTimes2Taosc,20) - + self.assertCheck("all_query_res2_taosc.txt", queryTimes2Taosc, 20) + # correct data testcase - self.assertfileDataTaosc("all_query_res0_taosc.txt","1604160000099") - self.assertfileDataTaosc("all_query_res1_taosc.txt","100") - self.assertfileDataTaosc("all_query_res2_taosc.txt","1604160000199") - + self.assertfileDataTaosc("all_query_res0_taosc.txt", "1604160000099") + self.assertfileDataTaosc("all_query_res1_taosc.txt", "100") + self.assertfileDataTaosc("all_query_res2_taosc.txt", "1604160000199") + # delete useless files - os.system("rm -rf ./query_res*") + os.system("rm -rf ./query_res*") os.system("rm -rf ./all_query*") - # use restful api to query - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertrestdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/queryRestful.json" % binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertrestdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryRestful.json" % + binPath) os.system("cat query_res0.txt* > all_query_res0_rest.txt") os.system("cat query_res1.txt* > all_query_res1_rest.txt") os.system("cat query_res2.txt* > all_query_res2_rest.txt") - - # correct Times testcases + + # correct Times testcases queryTimes0Restful = self.queryTimesRestful("all_query_res0_rest.txt") - self.assertCheck("all_query_res0_rest.txt",queryTimes0Restful,6) + self.assertCheck("all_query_res0_rest.txt", queryTimes0Restful, 6) queryTimes1Restful = self.queryTimesRestful("all_query_res1_rest.txt") - self.assertCheck("all_query_res1_rest.txt",queryTimes1Restful,6) - + self.assertCheck("all_query_res1_rest.txt", queryTimes1Restful, 6) + queryTimes2Restful = self.queryTimesRestful("all_query_res2_rest.txt") - self.assertCheck("all_query_res2_rest.txt",queryTimes2Restful,4) + self.assertCheck("all_query_res2_rest.txt", queryTimes2Restful, 4) # correct data testcase data0 = self.getfileDataRestful("all_query_res0_rest.txt") - self.assertCheck('all_query_res0_rest.txt',data0,"2020-11-01 00:00:00.009") + self.assertCheck( + 'all_query_res0_rest.txt', + data0, + "2020-11-01 00:00:00.009") data1 = self.getfileDataRestful("all_query_res1_rest.txt") - self.assertCheck('all_query_res1_rest.txt',data1,10) + self.assertCheck('all_query_res1_rest.txt', data1, 10) data2 = self.getfileDataRestful("all_query_res2_rest.txt") - self.assertCheck('all_query_res2_rest.txt',data2,"2020-11-01 00:00:00.004") - - + self.assertCheck( + 'all_query_res2_rest.txt', + data2, + "2020-11-01 00:00:00.004") + # query times less than or equal to 100 - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/querySpeciMutisql100.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/querySuperMutisql100.json" % binPath) - - #query result print QPS - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/queryQps.json" % binPath) - + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/querySpeciMutisql100.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/querySuperMutisql100.json" % + binPath) + + # query result print QPS + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryQps.json" % + binPath) + # use illegal or out of range parameters query json file - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - exceptcode = os.system("%staosdemo -f tools/taosdemoAllTest/queryTimes0.json" % binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + exceptcode = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryTimes0.json" % + binPath) assert exceptcode != 0 - exceptcode0 = os.system("%staosdemo -f tools/taosdemoAllTest/queryTimesless0.json" % binPath) + exceptcode0 = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryTimesless0.json" % + binPath) assert exceptcode0 != 0 - exceptcode1 = os.system("%staosdemo -f tools/taosdemoAllTest/queryConcurrentless0.json" % binPath) + exceptcode1 = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryConcurrentless0.json" % + binPath) assert exceptcode1 != 0 - exceptcode2 = os.system("%staosdemo -f tools/taosdemoAllTest/queryConcurrent0.json" % binPath) + exceptcode2 = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryConcurrent0.json" % + binPath) assert exceptcode2 != 0 - exceptcode3 = os.system("%staosdemo -f tools/taosdemoAllTest/querrThreadsless0.json" % binPath) + exceptcode3 = os.system( + "%staosdemo -f tools/taosdemoAllTest/querrThreadsless0.json" % + binPath) assert exceptcode3 != 0 - exceptcode4 = os.system("%staosdemo -f tools/taosdemoAllTest/querrThreads0.json" % binPath) + exceptcode4 = os.system( + "%staosdemo -f tools/taosdemoAllTest/querrThreads0.json" % + binPath) assert exceptcode4 != 0 # delete useless files os.system("rm -rf ./insert_res.txt") - os.system("rm -rf tools/taosdemoAllTest/*.py.sql") - os.system("rm -rf ./querySystemInfo*") - os.system("rm -rf ./query_res*") + os.system("rm -rf tools/taosdemoAllTest/*.py.sql") + os.system("rm -rf ./querySystemInfo*") + os.system("rm -rf ./query_res*") os.system("rm -rf ./all_query*") os.system("rm -rf ./test_query_res0.txt") - - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) + tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoTestTblAlt.py b/tests/pytest/tools/taosdemoTestTblAlt.py index 70df535f59cbb97469b7a73e4e230d9a8671bfc7..89c1b92e140cb1e19b549d3248693153e116c52e 100644 --- a/tests/pytest/tools/taosdemoTestTblAlt.py +++ b/tests/pytest/tools/taosdemoTestTblAlt.py @@ -54,27 +54,36 @@ class TDTestCase: binPath = buildPath + "/build/bin/" if(threadID == 0): - os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT -m t" % + print("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT" % + (binPath, self.numberOfTables, self.numberOfRecords)) + os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT" % (binPath, self.numberOfTables, self.numberOfRecords)) if(threadID == 1): time.sleep(2) print("use test") - while True: + max_try = 100 + count = 0 + while (count < max_try): try: tdSql.execute("use test") break except Exception as e: tdLog.info("use database test failed") - time.sleep(1) + time.sleep(2) + count += 1 + print("try %d times" % count) continue # check if all the tables have heen created - while True: + count = 0 + while (count < max_try): try: tdSql.query("show tables") except Exception as e: tdLog.info("show tables test failed") - time.sleep(1) + time.sleep(2) + count += 1 + print("try %d times" % count) continue rows = tdSql.queryRows @@ -83,13 +92,17 @@ class TDTestCase: break time.sleep(1) # check if there are any records in the last created table - while True: + count = 0 + while (count < max_try): print("query started") + print("try %d times" % count) try: - tdSql.query("select * from test.t7") + tdSql.query("select * from test.d7") except Exception as e: tdLog.info("select * test failed") time.sleep(2) + count += 1 + print("try %d times" % count) continue rows = tdSql.queryRows @@ -100,8 +113,8 @@ class TDTestCase: print("alter table test.meters add column c10 int") tdSql.execute("alter table test.meters add column c10 int") - print("insert into test.t7 values (now, 1, 2, 3, 4, 0)") - tdSql.execute("insert into test.t7 values (now, 1, 2, 3, 4, 0)") + print("insert into test.d7 values (now, 1, 2, 3, 4, 0)") + tdSql.execute("insert into test.d7 values (now, 1, 2, 3, 4, 0)") def run(self): tdSql.prepare() diff --git a/tests/pytest/tools/windows_input.py b/tests/pytest/tools/windows_input.py new file mode 100644 index 0000000000000000000000000000000000000000..99e30fa311673903850149c1916c6466c6a29ee5 --- /dev/null +++ b/tests/pytest/tools/windows_input.py @@ -0,0 +1,86 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +from uiautomation import WindowControl +from util.cases import * +from util.sql import * +import clipboard + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.host = conn._host + + def win_input_test(self): + os.system("start") + time.sleep(1) + + # 获取CMD窗口 + # window = DocumentControl(searchDepth=3, Name='Text Area') + window = WindowControl(searchDepth=1, AutomationId='Console Window') + # 切换英文输入法 + # window.SendKeys('\\') + # window.SendKeys('{Enter}') + # window.SendKeys('{Shift}') + # window.SendKeys('\\') + # window.SendKeys('{Enter}') + + # 切换目录 + window.SendKeys('c:') + window.SendKeys('{Enter}') + window.SendKeys('cd \\') + window.SendKeys('{Enter}') + window.SendKeys('cd c:\\TDengine') + window.SendKeys('{Enter}') + # 启动taos.exe + window.SendKeys('taos.exe -h %s || taos.exe' % (self.host)) + window.SendKeys('{Enter}') + # 输入 + + temp = '' + for i in range(300): + temp += 'a' + sql = "insert into db.tb values(now,'%s');" % temp + window.SendKeys(sql) + window.SendKeys('{Enter}') + window.SendKeys('{Ctrl}A') + window.SendKeys('{Ctrl}C') + # 获取剪切板里面的复制内容 + result = clipboard.paste() + window.SendKeys('{Ctrl}C') + window.SendKeys('exit') + window.SendKeys('{Enter}') + return result + + def run(self): + tdSql.prepare() + + ret = tdSql.execute('create table tb (ts timestamp, i binary(300))') + + result = self.win_input_test() + tdLog.info(result) + + tdSql.query("select * from tb") + tdSql.checkRows(1) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tt.py b/tests/pytest/tt.py new file mode 100644 index 0000000000000000000000000000000000000000..313a48c96bcc4d4669c5a7236308777a9974f847 --- /dev/null +++ b/tests/pytest/tt.py @@ -0,0 +1,125 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import threading +import taos +import sys +import json +import time +import random +import requests +import argparse +import datetime +import string +from requests.auth import HTTPBasicAuth + +class CompressTest: + # def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test', + # stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5, + # stableNum = 2,subtableNum = 1000,insertRows = 100): + def __init__(self): + self.host = 'vm95' + self.ts = 1569859200000 + self.dbname = 'db' + self.user = 'root' + self.password = 'taosdata' + self.loop = 6 + + def rest_query(self,sql): #rest 接口 + host = self.host + user = self.user + password = self.password + port =6041 + url = "http://{}:{}/rest/sql".format(host, port ) + try: + r = requests.post(url, + data = 'use %s' % self.dbname, + auth = HTTPBasicAuth('root', 'taosdata')) + r = requests.post(url, + data = sql, + auth = HTTPBasicAuth('root', 'taosdata')) + except: + print("REST API Failure (TODO: more info here)") + raise + rj = r.json() + if ('status' not in rj): + raise RuntimeError("No status in REST response") + + if rj['status'] == 'error': # clearly reported error + if ('code' not in rj): # error without code + raise RuntimeError("REST error return without code") + errno = rj['code'] # May need to massage this in the future + # print("Raising programming error with REST return: {}".format(rj)) + raise taos.error.ProgrammingError( + rj['desc'], errno) # todo: check existance of 'desc' + + if rj['status'] != 'succ': # better be this + raise RuntimeError( + "Unexpected REST return status: {}".format( + rj['status'])) + + nRows = rj['rows'] if ('rows' in rj) else 0 + return nRows + + + def query_thread_n(self,threadID): #使用原生python接口查询 + tstart = time.time() + host = self.host + user = self.user + password = self.password + conn = taos.connect( + host, + user, + password, + ) + cl = conn.cursor() + cl.execute("use %s;" % self.dbname) + print("Thread %d: starting" % threadID) + loop = self.loop + for loop in range(7): + ts = 1569859200000+loop*86400000 + try: + # sql = 'select ts,c0 from stb_%d where ts >%d and ts <%d' % (threadID, ts, ts+86400000) + sql = 'select ts,c1 from stb_%d where ts >%d and ts <%d' % (threadID, ts, ts+86400000) + # print("sql is ",sql) + # start = time.time() + cl.execute(sql) + cl.fetchall() + # end = time.time() + # print("time cost :",end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql),str(e))) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + cl.close() + conn.close() + tstop = time.time() + print("Thread %d: finishing, time collaps:%d " % (threadID, tstop - tstart)) + + + def run(self): + threads = [] + for i in range(10): + thread = threading.Thread(target=self.query_thread_n, args=(i,)) + threads.append(thread) + thread.start() + # for i in range(1,11): + # thread = threading.Thread(target=self.query_thread_r, args=(i,)) + # threads.append(thread) + # thread.start() + +q = CompressTest() +q.run() diff --git a/tests/pytest/update/update_options.py b/tests/pytest/update/update_options.py index dd1b82fc596a3a977b028234900337474b971ec2..f70ac6cc1d4299e481af30e2e9d24c3b24631856 100644 --- a/tests/pytest/update/update_options.py +++ b/tests/pytest/update/update_options.py @@ -63,6 +63,7 @@ class TDTestCase: tdLog.sleep(3) # test case for https://jira.taosdata.com:18080/browse/TS-402 + tdLog.info("test case for update option 1") tdSql.execute("create database test update 1") tdSql.execute("use test") @@ -75,7 +76,39 @@ class TDTestCase: tdSql.checkData(0, 2, None) tdSql.checkData(0, 3, 9) - + tdSql.execute("drop table if exists tb") + tdSql.execute("create table tb (ts timestamp, c1 int, c2 int, c3 int)") + tdSql.execute("insert into tb values(%d, 1, 2, 3)(%d, null, 4, 5)(%d, 6, null, 7)" % (self.ts, self.ts, self.ts)) + + tdSql.query("select * from tb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 6) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 3, 7) + + # https://jira.taosdata.com:18080/browse/TS-424 + tdLog.info("test case for update option 2") + tdSql.execute("create database db2 update 2") + tdSql.execute("use db2") + + tdSql.execute("create table tb (ts timestamp, c1 int, c2 int, c3 int)") + tdSql.execute("insert into tb values(%d, 1, 2, 3)(%d, null, null, 9)" % (self.ts, self.ts)) + + tdSql.query("select * from tb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 2) + tdSql.checkData(0, 3, 9) + + tdSql.execute("drop table if exists tb") + tdSql.execute("create table tb (ts timestamp, c1 int, c2 int, c3 int)") + tdSql.execute("insert into tb values(%d, 1, 2, 3)(%d, null, 4, 5)(%d, 6, null, 7)" % (self.ts, self.ts, self.ts)) + + tdSql.query("select * from tb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 6) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 7) def stop(self): diff --git a/tests/robust/robust.c b/tests/robust/robust.c index 3488ca1b9be8b6cca4d04fe5ba5841f00527b25d..20abef6b306041f94b3474cbe6c201783bf81a87 100644 --- a/tests/robust/robust.c +++ b/tests/robust/robust.c @@ -188,12 +188,12 @@ void insertImp(void *param) { return; } taos_free_result(result); - int64_t time = getTimeStampMs(); + int64_t test_time = getTimeStampMs(); while (true) { sqlLen = sprintf(sql, "insert into"); for (int i = (pThread->threadId - 1) * arguments.stable / arguments.client; i < pThread->threadId * arguments.stable / arguments.client; i++) { for (int j = 0; j < arguments.table; j++) { - sqlLen += sprintf(sql + sqlLen, " s%d_%d values (%ld, %d, %d, %d)", i, j, time, rand(), rand(), rand()); + sqlLen += sprintf(sql + sqlLen, " s%d_%d values (%ld, %d, %d, %d)", i, j, test_time, rand(), rand(), rand()); count++; if ( (1048576 - sqlLen) < 49151 || i == (pThread->threadId * arguments.stable / arguments.client - 1)) { result = taos_query(pThread->taos, sql); @@ -208,7 +208,7 @@ void insertImp(void *param) { } } } - time += 1000*arguments.interval; + test_time += 1000*arguments.interval; } } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 72bb9471db8e2c3043306c332c608f1b4f1df836..0281c4e9ee1554462dab9875c624beefa3ce8ad9 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -1761,7 +1761,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); - return -1; + exit(1); } int id = 0; @@ -1797,9 +1797,44 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { return 0; } +int stmt_multi_insert_check(TAOS_STMT *stmt) { + char *sql; + // The number of tag column list is not equal to the number of tag value list + sql = "insert into ? using stb1 (id1) tags(1,?) values(?,?,?,?,?,?,?,?,?,?)"; + if (0 == taos_stmt_prepare(stmt, sql, 0)) { + printf("failed to check taos_stmt_prepare. sql:%s\n", sql); + exit(1); + } + // The number of column list is not equal to the number of value list + sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') " + "(ts, b, v1, v2, v4, v8, f4, f8, bin) values(?,?,?,?,?,?,?,?,?,?)"; + if (0 == taos_stmt_prepare(stmt, sql, 0)) { + printf("failed to check taos_stmt_prepare. sql:%s\n", sql); + exit(1); + } + sql = "insert into ? using stb1 () tags(1,?) values(?,?,?,?,?,?,?,?,?,?)"; + if (0 == taos_stmt_prepare(stmt, sql, 0)) { + printf("failed to check taos_stmt_prepare. sql:%s\n", sql); + exit(1); + } + + sql = "insert into ? using stb1 ( tags(1,?) values(?,?,?,?,?,?,?,?,?,?)"; + if (0 == taos_stmt_prepare(stmt, sql, 0)) { + printf("failed to check taos_stmt_prepare. sql:%s\n", sql); + exit(1); + } + + sql = "insert into ? using stb1 ) tags(1,?) values(?,?,?,?,?,?,?,?,?,?)"; + if (0 == taos_stmt_prepare(stmt, sql, 0)) { + printf("failed to check taos_stmt_prepare. sql:%s\n", sql); + exit(1); + } + + return 0; +} //1 tables 10 records int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { @@ -1985,7 +2020,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { code = taos_stmt_set_tbname_tags(stmt, buf, NULL); if (code != 0){ printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt)); - return -1; + goto exit; } taos_stmt_bind_param_batch(stmt, params + id * 10); @@ -2002,6 +2037,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); +exit: free(v.ts); free(lb); free(params); @@ -2192,7 +2228,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); - return -1; + goto exit; //exit(1); } @@ -2220,6 +2256,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); +exit: free(v.ts); free(lb); free(params); @@ -2429,7 +2466,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { code = taos_stmt_bind_param_batch(stmt, params + id * 10); if (code != 0) { printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; + goto exit; } taos_stmt_add_batch(stmt); @@ -2445,6 +2482,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); +exit: free(v.ts); free(lb); free(params); @@ -2656,7 +2694,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { code = taos_stmt_bind_param_batch(stmt, params + id * 10); if (code != 0) { printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; + goto exit; } taos_stmt_add_batch(stmt); @@ -2672,6 +2710,8 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + +exit: free(v.ts); free(lb); free(params); @@ -2683,6 +2723,38 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { } +int stmt_funcb_autoctb_e6(TAOS_STMT *stmt) { + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(now,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + } + + return 0; +} + + +int stmt_funcb_autoctb_e7(TAOS_STMT *stmt) { + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,true,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + } + + return 0; +} + + +int stmt_funcb_autoctb_e8(TAOS_STMT *stmt) { + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,1,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("case success:failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + } + + return 0; +} + //300 tables 60 records int stmt_funcb1(TAOS_STMT *stmt) { @@ -4505,7 +4577,6 @@ void* runcase(void *par) { (void)idx; - #if 1 prepare(taos, 1, 1); @@ -4820,6 +4891,54 @@ void* runcase(void *par) { #endif +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("e6 start\n"); + stmt_funcb_autoctb_e6(stmt); + printf("e6 end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("e7 start\n"); + stmt_funcb_autoctb_e7(stmt); + printf("e7 end\n"); + taos_stmt_close(stmt); + +#endif + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("e8 start\n"); + stmt_funcb_autoctb_e8(stmt); + printf("e8 end\n"); + taos_stmt_close(stmt); + +#endif + + +#if 1 + prepare(taos, 1, 0); + + stmt = taos_stmt_init(taos); + + printf("stmt_multi_insert_check start\n"); + stmt_multi_insert_check(stmt); + printf("stmt_multi_insert_check end\n"); + taos_stmt_close(stmt); +#endif + #if 1 prepare(taos, 1, 1); @@ -5007,7 +5126,6 @@ void* runcase(void *par) { printf("check result end\n"); #endif - #if 1 preparem(taos, 0, idx); diff --git a/tests/script/api/stmt.c b/tests/script/api/stmt.c index baf40c1421df1de4afcc8570288f642df067130a..ef62b22f9a52c226c0194173fd391ecb782c9e6b 100644 --- a/tests/script/api/stmt.c +++ b/tests/script/api/stmt.c @@ -247,16 +247,16 @@ void taos_stmt_bind_param_test() { void taos_stmt_bind_single_param_batch_test() { printf("start taos_stmt_bind_single_param_batch test\n"); TAOS_STMT * stmt = NULL; - TAOS_MULTI_BIND *bind = NULL; - assert(taos_stmt_bind_single_param_batch(stmt, bind, 0) != 0); + TAOS_MULTI_BIND *test_bind = NULL; + assert(taos_stmt_bind_single_param_batch(stmt, test_bind, 0) != 0); printf("finish taos_stmt_bind_single_param_batch test\n"); } void taos_stmt_bind_param_batch_test() { printf("start taos_stmt_bind_param_batch test\n"); TAOS_STMT * stmt = NULL; - TAOS_MULTI_BIND *bind = NULL; - assert(taos_stmt_bind_param_batch(stmt, bind) != 0); + TAOS_MULTI_BIND *test_bind = NULL; + assert(taos_stmt_bind_param_batch(stmt, test_bind) != 0); printf("finish taos_stmt_bind_param_batch test\n"); } diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 37b1b4d973c26c1b2b37531f25f4d75b83b1f6c6..34b426752ab0953fec0f648c01c6effde317342b 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -223,3 +223,4 @@ run general/db/show_create_db.sim run general/db/show_create_table.sim run general/parser/like.sim run general/parser/interp_blocks.sim +run general/parser/create_tb_with_timestamp_tag.sim diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 17ae6cfd6b8b5636101e67e8d99f6999e50a06a5..2576b1442a7793ab4bb5d22d52ce950d885cdbd0 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -553,19 +553,19 @@ endi sql_error select first(c6) - last(c6) *12 / count(*) from $stb group by c3; sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5; -if $rows != 10 then +if $rows != 11 then return -1 endi -if $data00 != 0.000000000 then +if $data00 != -0.002160000 then return -1 endi -if $data10 != 0.997600000 then +if $data10 != 0.000000000 then return -1 endi -if $data90 != 8.978400000 then +if $data90 != 7.980800000 then return -1 endi diff --git a/tests/script/general/parser/create_tb_with_timestamp_tag.sim b/tests/script/general/parser/create_tb_with_timestamp_tag.sim new file mode 100644 index 0000000000000000000000000000000000000000..452fd2ddbf7d475a00f2fdd334e5065b6da71dc9 --- /dev/null +++ b/tests/script/general/parser/create_tb_with_timestamp_tag.sim @@ -0,0 +1,115 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db precision 'ns' +sql use $db + +sql create stable st1 (ts timestamp , c1 int) tags(t1 timestamp, t2 int) + +sql create table t1_0 using st1 tags(now, 0) +#nanoseconds +sql create table t1_1 using st1 tags(now + 1b, 0) +#microseconds +sql create table t1_2 using st1 tags(now + 1u, 0) +#milliseconds +sql create table t1_3 using st1 tags(now + 1a, 0) +#seconds +sql create table t1_4 using st1 tags(now + 1s, 0) +#minutes +sql create table t1_5 using st1 tags(now + 1m, 0) +#hours +sql create table t1_6 using st1 tags(now + 1h, 0) +#days +sql create table t1_7 using st1 tags(now + 1d, 0) +#weeks +sql create table t1_8 using st1 tags(now + 1w, 0) +#months(not supported) +sql_error create table t1_9 using st1 tags(now + 1n, 0) +#years(not supported) +sql_error create table t1_10 using st1 tags(now + 1y, 0) + + +sql create stable st2 (ts timestamp , c1 int) tags(t1 timestamp, t2 int) + +sql create table t2_0 using st2 tags(now, 0) +#nanoseconds +sql create table t2_1 using st2 tags(now - 1b, 0) +#microseconds +sql create table t2_2 using st2 tags(now - 1u, 0) +#milliseconds +sql create table t2_3 using st2 tags(now - 1a, 0) +#seconds +sql create table t2_4 using st2 tags(now - 1s, 0) +#minutes +sql create table t2_5 using st2 tags(now - 1m, 0) +#hours +sql create table t2_6 using st2 tags(now - 1h, 0) +#days +sql create table t2_7 using st2 tags(now - 1d, 0) +#weeks +sql create table t2_8 using st2 tags(now - 1w, 0) +#months(not supported) +sql_error create table t2_9 using st2 tags(now - 1n, 0) +#years(not supported) +sql_error create table t2_10 using st2 tags(now - 1y, 0) + +sql insert into t1_0 values (now, 0) +sql insert into t1_1 values (now, 1) +sql insert into t1_2 values (now, 2) +sql insert into t1_3 values (now, 3) +sql insert into t1_4 values (now, 4) +sql insert into t1_5 values (now, 5) +sql insert into t1_6 values (now, 6) +sql insert into t1_7 values (now, 7) +sql insert into t1_8 values (now, 8) + +sql insert into t2_0 values (now, 0) +sql insert into t2_1 values (now, 1) +sql insert into t2_2 values (now, 2) +sql insert into t2_3 values (now, 3) +sql insert into t2_4 values (now, 4) +sql insert into t2_5 values (now, 5) +sql insert into t2_6 values (now, 6) +sql insert into t2_7 values (now, 7) +sql insert into t2_8 values (now, 8) + +sql select * from st1 + +if $rows != 9 then + return -1 +endi + +sql select * from st2 + +if $rows != 9 then + return -1 +endi + +sql create stable st3 (ts timestamp , c1 int) tags (t1 timestamp, t2 timestamp, t3 timestamp, t4 timestamp, t5 timestamp, t6 timestamp, t7 timestamp, t8 timestamp, t9 timestamp) +sql create table t3 using st3 tags(now, now + 1b, now + 1u, now + 1a, now + 1s, now + 1m, now + 1h, now + 1d, now + 1w) +sql insert into t3 values (now, 1) + +sql select * from st3 +if $rows != 1 then + return -1 +endi + +sql create stable st4 (ts timestamp , c1 int) tags (t1 timestamp, t2 timestamp, t3 timestamp, t4 timestamp, t5 timestamp, t6 timestamp, t7 timestamp, t8 timestamp, t9 timestamp) +sql create table t4 using st4 tags(now, now - 1b, now - 1u, now - 1a, now - 1s, now - 1m, now - 1h, now - 1d, now - 1w) +sql insert into t4 values (now, 1) + +sql select * from st4 +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 48e52cc7d7f946a18c366fc1ba667daf3fcf55eb..94f75962b4118720f2f4bdec82f4fbbe47d19cea 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -1197,3 +1197,16 @@ if $data00 != 0.000000000 then return -1 endi +sql select derivative(aa,1m,0) from (select avg(voltage) as aa from smeters where ts>='2021-08-08 10:10:10.000' and ts < '2021-08-08 10:10:20.000' and current=10 interval(1000a)); +if $rows != 1 then + return -1 +endi +if $data00 != @21-08-08 10:10:12.000@ then + return -1 +endi +if $data01 != 0.000000000 then + return -1 +endi + + + diff --git a/tests/script/general/parser/like.sim b/tests/script/general/parser/like.sim new file mode 100644 index 0000000000000000000000000000000000000000..3b1fbbf83e7ca0d08bb324a72dd7046463d8fe61 --- /dev/null +++ b/tests/script/general/parser/like.sim @@ -0,0 +1,100 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 10 +sql connect +print ======================== dnode1 start + + +$db = testdb +sql drop database if exists $db +sql create database $db cachelast 2 +sql use $db + +$table1 = table_name +$table2 = tablexname + +sql create table $table1 (ts timestamp, b binary(20)) +sql create table $table2 (ts timestamp, b binary(20)) + +sql insert into $table1 values(now, "table_name") +sql insert into $table1 values(now-1m, "tablexname") +sql insert into $table1 values(now-2m, "tablexxx") +sql insert into $table1 values(now-3m, "table") + +sql select b from $table1 +if $rows != 4 then + return -1 +endi + +sql select b from $table1 where b like 'table_name' +if $rows != 2 then + return -1 +endi + + +sql select b from $table1 where b like 'table\_name' +if $rows != 1 then + return -1 +endi + +sql show tables; +if $rows != 2 then + return -1 +endi + +sql show tables like 'table_name' +if $rows != 2 then + return -1 +endi + +sql show tables like 'table\_name' +if $rows != 1 then + return -1 +endi + +sql create database escape_percentage; +sql use escape_percentage; +$table1 = tablename +$table2 = table2 + +sql create table $table1 (ts timestamp, b binary(20)) +sql create table $table2 (ts timestamp, b binary(20)) + +sql insert into $table1 values(now, "table%name") +sql insert into $table1 values(now-1m, "table%") +sql insert into $table1 values(now-2m, "table%%%") +sql insert into $table1 values(now-3m, "table") + +sql select b from $table1 where b like 'table\%' +print $rows +if $rows != 1 then + return -1 +endi +sql select b from $table1 where b like 'table\%\%\%' +print $rows +if $rows != 1 then + return -1 +endi +sql select b from $table1 where b like 'table%' +print $rows +if $rows != 4 then + return -1 +endi +sql show tables like 'table\%' +print $rows +if $rows != 0 then + return -1 +endi +sql show tables like 'table%' +print $rows +if $rows != 2 then + return -1 +endi +sql drop database escape_percentage +system sh/exec.sh -n dnode1 -s stop -x SIGINT + + diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index 4182b867053d5d26cdc95970c545fd0c5f20d09c..5bbdace4da0cb65b4573e2f9cd93b53ac28f6c2f 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -330,9 +330,9 @@ if $data12 != 71680.000000000 then return -1 endi -sql select top(x, 20) from (select c1 x from nest_tb0); +sql select top(x, 20) from (select ts,c1 x from nest_tb0); -sql select bottom(x, 20) from (select c1 x from nest_tb0) +sql select bottom(x, 20) from (select ts,c1 x from nest_tb0) print ===================> group by + having @@ -407,12 +407,14 @@ if $data03 != @20-09-15 00:00:00.000@ then return -1 endi -sql select diff(val) from (select c1 val from nest_tb0); +sql_error select diff(val) from (select c1 val from nest_tb0); + +sql select diff(val) from (select ts,c1 val from nest_tb0); if $rows != 9999 then return -1 endi -if $data00 != @70-01-01 08:00:00.000@ then +if $data00 != @20-09-15 00:01:00.000@ then return -1 endi @@ -884,5 +886,18 @@ if $data00 != 24 then return -1 endi +sql select sum(a)/sum(b) from meters where ts >= '2021-09-30 15:00:00.000' and ts <= '2021-09-30 15:00:05.000' interval(1s) fill(null) group by area order by area; +if $rows != 12 then + return -1 +endi +if $data00 != @21-09-30 15:00:00.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 0 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/slimit_query.sim b/tests/script/general/parser/slimit_query.sim index 0a793f0611a5875f53ac75644f3156ad9aa6cc65..0dbcffa0a43b596d542c8799a9a94170e6f71f5a 100644 --- a/tests/script/general/parser/slimit_query.sim +++ b/tests/script/general/parser/slimit_query.sim @@ -31,6 +31,8 @@ $tsu = $tsu + $ts0 #sql_error select top(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu slimit 5 offset 1 #sql_error select bottom(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu slimit 5 offset 1 +sql_error select t1 from $stb slimit 5 offset 1; + ### select from stb + group by + slimit offset sql select max(c1), min(c2), avg(c3), sum(c4), spread(c5), sum(c6), count(c7), first(c8), last(c9) from $stb group by t1 slimit 5 soffset 0 if $rows != 5 then diff --git a/tests/script/general/parser/udf_dll.sim b/tests/script/general/parser/udf_dll.sim index 0f9436762adb645785ddcf9a4abaf4a5be810a34..cdbdfd815ed8f4fec5d17649f84b2e9837bf5813 100644 --- a/tests/script/general/parser/udf_dll.sim +++ b/tests/script/general/parser/udf_dll.sim @@ -10,7 +10,7 @@ sql connect print ======================== dnode1 start sql create function add_one as '/tmp/add_one.so' outputtype int; -sql create aggregate function sum_double as '/tmp/sum_double.so' outputtype int; +sql create aggregate function sum_double as '/tmp/sum_double.so' outputtype bigint; sql show functions; if $rows != 2 then return -1 @@ -452,6 +452,7 @@ if $data31 != 2 then return -1 endi +sql_error select add_one(f1) from tb1 order by ts desc; sql select add_one(f1) from tb1 limit 2; if $rows != 2 then diff --git a/tests/script/general/parser/udf_dll_stable.sim b/tests/script/general/parser/udf_dll_stable.sim index b8da57467e912ff27f4fbda7226c75e089f04808..07784d794d2fbdf7203045e688db27735c590af3 100644 --- a/tests/script/general/parser/udf_dll_stable.sim +++ b/tests/script/general/parser/udf_dll_stable.sim @@ -10,9 +10,10 @@ sql connect print ======================== dnode1 start sql create function add_one as '/tmp/add_one.so' outputtype int; -sql create aggregate function sum_double as '/tmp/sum_double.so' outputtype int; +sql create function add_one_64232 as '/tmp/add_one_64232.so' outputtype int; +sql create aggregate function sum_double as '/tmp/sum_double.so' outputtype bigint; sql show functions; -if $rows != 2 then +if $rows != 3 then return -1 endi @@ -1153,6 +1154,92 @@ if $data61 != 22 then return -1 endi +sql_error select sum_double(f1),add_one(f1) from tb1 where ts>="2021-03-23 17:00:00.000" and ts<="2021-03-24 20:00:00.000" interval (1h) sliding (30m); + +sql select add_one(f1) from (select * from tb1); +if $rows != 7 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 5 then + return -1 +endi +if $data40 != 6 then + return -1 +endi +if $data50 != 7 then + return -1 +endi +if $data60 != 8 then + return -1 +endi + +sql select add_one(ff1) from (select add_one(f1) as ff1 from tb1); +if $rows != 7 then + return -1 +endi + +if $data00 != 3 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 6 then + return -1 +endi +if $data40 != 7 then + return -1 +endi +if $data50 != 8 then + return -1 +endi +if $data60 != 9 then + return -1 +endi + +sql_error select add_one(f1),sub_one(f1) from tb1; + + +sql create table taaa (ts timestamp, f1 bigint); +sql insert into taaa values (now, 1); +sleep 100 +sql insert into taaa values (now, 10); +sleep 100 +sql insert into taaa values (now, 1000); +sleep 100 +sql insert into taaa values (now, 100); + +sql select add_one_64232(f1) from taaa; +if $rows != 4 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi +if $data10 != 11 then + return -1 +endi +if $data20 != 1001 then + return -1 +endi +if $data30 != 101 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b4aad278d8836388990f4b11313db985e8ee05e6..2709c3cddab543714dd77c91aa116c50d8cba713 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -396,7 +396,7 @@ cd ../../../debug; make ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim -./test.sh -f unique/arbitrator/insert_duplicationTs.sim +#./test.sh -f unique/arbitrator/insert_duplicationTs.sim ./test.sh -f general/parser/join_manyblocks.sim ./test.sh -f general/parser/stableOp.sim ./test.sh -f general/parser/timestamp.sim @@ -407,4 +407,8 @@ cd ../../../debug; make ./test.sh -f general/parser/last_cache.sim ./test.sh -f unique/big/balance.sim +./test.sh -f general/parser/udf.sim +./test.sh -f general/parser/udf_dll.sim +./test.sh -f general/parser/udf_dll_stable.sim + #======================b7-end=============== diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c index cd8ba0ff15c135bdf845af57e39d5085c0fbcb20..9faea60374766e47f82a70880fe2b1376bbc12dc 100644 --- a/tests/script/sh/abs_max.c +++ b/tests/script/sh/abs_max.c @@ -1,11 +1,12 @@ #include #include #include +#include typedef struct SUdfInit{ int maybe_null; /* 1 if function can return NULL */ int decimals; /* for real functions */ - long long length; /* For string functions */ + int64_t length; /* For string functions */ char *ptr; /* free pointer for function data */ int const_item; /* 0 if result is independent of arguments */ } SUdfInit; @@ -14,30 +15,37 @@ typedef struct SUdfInit{ #define TSDB_DATA_INT_NULL 0x80000000L #define TSDB_DATA_BIGINT_NULL 0x8000000000000000L -void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, +void abs_max(char* data, short itype, short ibytes, int numOfRows, int64_t* ts, char* dataOutput, char* interBuf, char* tsOutput, int* numOfOutput, short otype, short obytes, SUdfInit* buf) { int i; - int r = 0; - printf("abs_max input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); + int64_t r = 0; + // printf("abs_max input data:%p, type:%d, rows:%d, ts:%p, %" PRId64 ", dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); if (itype == 5) { - r=*(long *)dataOutput; + r=*(int64_t *)dataOutput; *numOfOutput=0; for(i=0;i r) { r = v; } } - *(long *)dataOutput=r; + *(int64_t *)dataOutput=r; - printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); + // printf("abs_max out, dataoutput:%" PRId64", numOfOutput:%d\n", *(int64_t *)dataOutput, *numOfOutput); + }else { + *numOfOutput=0; } } @@ -45,44 +53,43 @@ void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) { int i; - int r = 0; - printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf); - *numOfOutput=1; - printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); + //int64_t r = 0; + // printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf); + // *numOfOutput=1; + // printf("abs_max finalize, dataoutput:%" PRId64", numOfOutput:%d\n", *(int64_t *)dataOutput, *numOfOutput); } void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { - int r = 0; + int64_t r = 0; if (numOfRows > 0) { - r = *((long *)data); + r = *((int64_t *)data); } - printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); + // printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); for (int i = 1; i < numOfRows; ++i) { - printf("abs_max_merge %d - %ld\n", i, *((long *)data + i)); - if (*((long*)data + i) > r) { - r= *((long*)data + i); + // printf("abs_max_merge %d - %" PRId64"\n", i, *((int64_t *)data + i)); + if (*((int64_t*)data + i) > r) { + r= *((int64_t*)data + i); } } - *(long*)dataOutput=r; + *(int64_t*)dataOutput=r; if (numOfRows > 0) { *numOfOutput=1; } else { *numOfOutput=0; } - printf("abs_max_merge, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); + // printf("abs_max_merge, dataoutput:%" PRId64", numOfOutput:%d\n", *(int64_t *)dataOutput, *numOfOutput); } int abs_max_init(SUdfInit* buf) { - printf("abs_max init\n"); + // printf("abs_max init\n"); return 0; } void abs_max_destroy(SUdfInit* buf) { - printf("abs_max destroy\n"); -} - + // printf("abs_max destroy\n"); +} \ No newline at end of file diff --git a/tests/script/sh/add_one.c b/tests/script/sh/add_one.c index e12cf8f26f6ddad67f9f7b091c033de46a3f6f50..de2a7d168c88d1a13bf5cbe6a327ea451ddb2546 100644 --- a/tests/script/sh/add_one.c +++ b/tests/script/sh/add_one.c @@ -14,20 +14,18 @@ void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts int* numOfOutput, short otype, short obytes, SUdfInit* buf) { int i; int r = 0; - printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); + // printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); if (itype == 4) { for(i=0;i +#include +#include + +typedef struct SUdfInit{ + int maybe_null; /* 1 if function can return NULL */ + int decimals; /* for real functions */ + long long length; /* For string functions */ + char *ptr; /* free pointer for function data */ + int const_item; /* 0 if result is independent of arguments */ +} SUdfInit; + +void add_one_64232(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput, + int* numOfOutput, short otype, short obytes, SUdfInit* buf) { + int i; + int r = 0; + printf("add_one_64232 input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); + if (itype == 5) { + for(i=0;i +#include +#include + +typedef struct SUdfInit{ + int maybe_null; /* 1 if function can return NULL */ + int decimals; /* for real functions */ + long long length; /* For string functions */ + char *ptr; /* free pointer for function data */ + int const_item; /* 0 if result is independent of arguments */ +} SUdfInit; + +void sub_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput, + int* numOfOutput, short otype, short obytes, SUdfInit* buf) { + int i; + int r = 0; + printf("sub_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); + if (itype == 4) { + for(i=0;i #include #include +#include typedef struct SUdfInit{ int maybe_null; /* 1 if function can return NULL */ int decimals; /* for real functions */ - long long length; /* For string functions */ + int64_t length; /* For string functions */ char *ptr; /* free pointer for function data */ int const_item; /* 0 if result is independent of arguments */ } SUdfInit; @@ -13,13 +14,13 @@ typedef struct SUdfInit{ #define TSDB_DATA_INT_NULL 0x80000000L -void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, +void sum_double(char* data, short itype, short ibytes, int numOfRows, int64_t* ts, char* dataOutput, char* interBuf, char* tsOutput, int* numOfOutput, short otype, short obytes, SUdfInit* buf) { int i; - int r = 0; - printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); + int64_t r = 0; + printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%"PRId64", dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); if (itype == 4) { - r=*(int *)dataOutput; + r=*(int64_t *)dataOutput; *numOfOutput=0; for(i=0;iptr)=*(int*)dataOutput*2; - *(int*)dataOutput=*(int*)(buf->ptr); - printf("sum_double finalize, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput); + int64_t r = 0; + // printf("sum_double_finalize dataoutput:%p:%"PRId64", numOfOutput:%d, buf:%p\n", dataOutput, *(int64_t*)dataOutput, *numOfOutput, buf); + // *numOfOutput=1; + *(int64_t*)(buf->ptr)=*(int64_t*)dataOutput*2; + *(int64_t*)dataOutput=*(int64_t*)(buf->ptr); + // printf("sum_double finalize, dataoutput:%"PRId64", numOfOutput:%d\n", *(int64_t *)dataOutput, *numOfOutput); } -void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { +void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int* numOfOutput, SUdfInit* buf) { int r = 0; - int sum = 0; + int64_t sum = 0; - printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); + // printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); for (int i = 0; i < numOfRows; ++i) { - printf("sum_double_merge %d - %d\n", i, *((int*)data + i)); - sum +=*((int*)data + i); + // printf("sum_double_merge %d - %"PRId64"\n", i, *((int64_t*)data + i)); + sum +=*((int64_t*)data + i); } - *(int*)dataOutput+=sum; + *(int64_t*)dataOutput+=sum; if (numOfRows > 0) { *numOfOutput=1; } else { *numOfOutput=0; } - printf("sum_double_merge, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput); + // printf("sum_double_merge, dataoutput:%"PRId64", numOfOutput:%d\n", *(int64_t *)dataOutput, *numOfOutput); } int sum_double_init(SUdfInit* buf) { buf->maybe_null=1; - buf->ptr = malloc(sizeof(int)); - printf("sum_double init\n"); + buf->ptr = malloc(sizeof(int64_t)); + // printf("sum_double init\n"); return 0; } void sum_double_destroy(SUdfInit* buf) { free(buf->ptr); - printf("sum_double destroy\n"); -} - + // printf("sum_double destroy\n"); +} \ No newline at end of file diff --git a/tests/script/unique/dnode/alternativeRole.sim b/tests/script/unique/dnode/alternativeRole.sim index 955b757f06df22c884565d0fac350456e79cb73a..7e647925d1d3d66d21f279ace852e3fc12496510 100644 --- a/tests/script/unique/dnode/alternativeRole.sim +++ b/tests/script/unique/dnode/alternativeRole.sim @@ -32,33 +32,48 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sleep 3000 +$x = 0 +show1: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + sql show dnodes print dnode1 $data5_1 -print dnode1 $data5_2 -print dnode1 $data5_3 +print dnode2 $data5_2 +print dnode3 $data5_3 if $data5_1 != mnode then - return -1 + goto show1 endi if $data5_2 != vnode then - return -1 + goto show1 endi if $data5_3 != any then - return -1 + goto show1 endi +show2: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 if $data2_1 != master then - return -1 + goto show2 endi if $data2_2 != null then - return -1 + goto show2 endi if $data2_3 != slave then - return -1 + goto show2 endi print ========== step2 @@ -72,26 +87,28 @@ sql create table d1.t6 (ts timestamp, i int) sql create table d1.t7 (ts timestamp, i int) sql create table d1.t8 (ts timestamp, i int) +show3: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + sql show dnodes print dnode1 $data2_1 print dnode2 $data2_2 print dnode3 $data2_3 if $data2_1 != 0 then - return -1 + goto show3 endi if $data2_2 != 1 then - return -1 + goto show3 endi if $data2_3 != 1 then - return -1 + goto show3 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 2e19dde3d9c52c20705d131f471a2e0e389589e4..d9a7e13138b125f92d0611614f74abf268eaee70 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -135,7 +135,7 @@ typedef struct _script_t { int32_t numOfLines; // number of lines in the script int32_t bgScriptLen; char fileName[MAX_FILE_NAME_LEN]; // script file name - char error[MAX_ERROR_LEN]; + char error[TSDB_MAX_BINARY_LEN + 100]; char * optionBuffer; SCmdLine *lines; // command list SVariable variables[MAX_VAR_LEN]; @@ -178,4 +178,4 @@ bool simExecuteLineInsertCmd(SScript *script, char *option); bool simExecuteLineInsertErrorCmd(SScript *script, char *option); void simVisuallizeOption(SScript *script, char *src, char *dst); -#endif \ No newline at end of file +#endif diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 397accfea57fec92dbe2b7f9b5c4b730a91e9cbd..843335406c60db688649daf452286420d09928f1 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -143,13 +143,13 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return var->varValue; } -int32_t simExecuteExpression(SScript *script, char *exp) { +int32_t simExecuteExpression(SScript *script, char *sim_exp) { char * op1, *op2, *var1, *var2, *var3, *rest; int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; char t0[1024], t1[1024], t2[1024], t3[2048]; int32_t result; - rest = paGetToken(exp, &var1, &var1Len); + rest = paGetToken(sim_exp, &var1, &var1Len); rest = paGetToken(rest, &op1, &op1Len); rest = paGetToken(rest, &var2, &var2Len); rest = paGetToken(rest, &op2, &op2Len); diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 1acdcd2ac6eb0ecb66e2977dee7577393ed242ef..4dc189d20e235ff70df5ebe76faaa50fc3e20116 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -251,11 +251,11 @@ SScript *simParseScript(char *fileName) { return script; } -int32_t simCheckExpression(char *exp) { +int32_t simCheckExpression(char *sim_exp) { char * op1, *op2, *op, *rest; int32_t op1Len, op2Len, opLen; - rest = paGetToken(exp, &op1, &op1Len); + rest = paGetToken(sim_exp, &op1, &op1Len); if (op1Len == 0) { sprintf(parseErr, "expression is required"); return -1; @@ -295,10 +295,10 @@ int32_t simCheckExpression(char *exp) { rest = paGetToken(rest, &op, &opLen); - if (opLen == 0) return (int32_t)(rest - exp); + if (opLen == 0) return (int32_t)(rest - sim_exp); /* if it is key word "then" */ - if (strncmp(op, "then", 4) == 0) return (int32_t)(op - exp); + if (strncmp(op, "then", 4) == 0) return (int32_t)(op - sim_exp); rest = paGetToken(rest, &op2, &op2Len); if (op2Len == 0) { @@ -312,7 +312,7 @@ int32_t simCheckExpression(char *exp) { } if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') { - return (int32_t)(rest - exp); + return (int32_t)(rest - sim_exp); } return -1; diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 0879e371ef62fee81786728e2b980442567fbaa1..1ee7d94b915e6db3881f359b6f14565ddc55f450 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -43,9 +43,9 @@ char *simParseArbitratorName(char *varName) { char *simParseHostName(char *varName) { static char hostName[140]; - int32_t index = atoi(varName + 8); + int32_t sim_index = atoi(varName + 8); int32_t port = 7100; - switch (index) { + switch (sim_index) { case 1: port = 7100; break;